diff --git a/CMakeLists.txt b/CMakeLists.txt index ae90c8bd74c..5ed6ad8a06f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ project(cime_tests Fortran C) # We rely on pio for cmake utilities like findnetcdf.cmake, so that we don't # need to duplicate this cmake code -list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/externals/pio2/cmake") +list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/src/externals/pio2/cmake") list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) include(CIME_utils) @@ -19,7 +19,7 @@ include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS}) # ------------------------------------------------------------------------ # Build mct # ------------------------------------------------------------------------ -set(MCT_ROOT "${CIME_ROOT}/externals/mct") +set(MCT_ROOT "${CIME_ROOT}/src/externals/mct") ExternalProject_add(mct_project PREFIX ${CMAKE_CURRENT_BINARY_DIR} @@ -58,14 +58,14 @@ link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) # csm_share (we don't build it here because it seems to be built differently # by different tests?) -set(SHARE_ROOT "${CIME_ROOT}/share/csm_share") -add_subdirectory(${SHARE_ROOT}/shr csm_share) +set(SHARE_ROOT "${CIME_ROOT}/src/share") +add_subdirectory(${SHARE_ROOT}/util csm_share) include_directories(${SHARE_ROOT}/include) # esmf_wrf_timemgr not built here because it depends on csm_share. -add_subdirectory(${CIME_ROOT}/share/esmf_wrf_timemgr esmf_wrf_timemgr) -include_directories(${CIME_ROOT}/share/esmf_wrf_timemgr) +add_subdirectory(${SHARE_ROOT}/esmf_wrf_timemgr esmf_wrf_timemgr) +include_directories(${SHARE_ROOT}/esmf_wrf_timemgr) # Now the actual test directories. -add_subdirectory(driver_cpl/unit_test) -add_subdirectory(share/csm_share/test/unit) +add_subdirectory(${CIME_ROOT}/src/drivers/mct/unit_test) +add_subdirectory(${SHARE_ROOT}/test/unit) diff --git a/ChangeLog b/ChangeLog index f769eb097e5..e26826d9437 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,3180 @@ ====================================================================== +Originator:Chris Fischer +Date: 4-4-2017 +Tag: cime5.3.0-alpha.06 +Answer Changes: None +Tests: code_checker, NODEFAIL +Dependencies: + +Brief Summary: + - Fix handling of spare nodes + +User interface changes: + +PR summary: git log --oneline --first-parent [previous_tag]..master +95019a3 Merge pull request #1302 from ESMCI/jgfouca/fixes_to_recent_prs + +Modified files: git diff --name-status [previous_tag] +M scripts/lib/CIME/XML/env_mach_pes.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py + +====================================================================== + +Originator: Chris Fischer +Date: 4-4-2017 +Tag: cime5.3.0-alpha.05 +Answer Changes: None +Tests: scripts_regression_test, code checker +Dependencies: + +Brief Summary: + - First try to support titan restart. + - Pull back ACME repo changes of CIME since last subtree split (late + January?) up to March 29 back into CIME master. + - Allow a custom input root through create_test. + - Update pio stubs to include new derived field needed in pio_init. + - Set a default value for esmf_logging so it does not have to appear in + drv_in. + - Add logging options to ESMF_Initialize + - Fix pylint issues. + - Fix manage_case --query-grids for cesm. + - Adds support for namelist entries of the form foo(3) = 'a' + - Add mpilib and compiler command line options to configure. + - summarize_cprnc_diffs: Print unnormalized RMS diffs. + - Make sure pio_typename is valid. + - New hooks for runnning external scripts for pre-run and post-run. + - Enable PIO rearranger options for PIO2. + - code_checker needs to work outside of ESMCI/CIME. + - Fix pylint error in buildnml. + - Add flags to make create_newcase non-interactive. + - Added mct/cime_config/namelist_definition_drv_flds.xml with updated + schema. + - Some driver code no longer needed to do this for cam. + - Supress a future warning on the use of condition in expect subroutine. + - Couple fixes to recent PR that added SUBMIT phase. + +User interface changes: + - Add --input-dir argument to create_test + - esmf_logging (and therefore drv_in) are now optional + - New run time XML variable: ESMF_LOGFILE_KIND + - New --answer flag to create_newcase + - Removes a distracting warning + +PR summary: git log --oneline --first-parent [previous_tag]..master +eeb1e2c Merge pull request #1275 from ESMCI/jgfouca/titan_restart +fbf1b0c Merge pull request #1287 from ESMCI/agsalin/merge-from-acme-03292017 +dda631e Merge pull request #1298 from ESMCI/jgfouca/allow_custom_input_root +2dc9a8a Merge pull request #1301 from jedwards4b/unit_test_stubs_update +3d76b10 Merge pull request #1300 from gold2718/esmfLog +0d320ec Merge pull request #1289 from gold2718/esmfLog +8890873 Merge pull request #1290 from ESMCI/jgfouca/grids_pylint_fixes +55a084b Merge pull request #1285 from ESMCI/mvertens/manage_case +9d8396a Merge pull request #1259 from jedwards4b/index_in_namelist +4e125ee Merge pull request #1268 from jedwards4b/configure_update +9759e15 Merge pull request #1279 from billsacks/summarize_rms +840ae3c Merge pull request #1273 from jedwards4b/pio_typename_fix +07e61ed Merge pull request #1256 from ESMCI/mvertens/phys_cycle +85f831d Merge pull request #1260 from ESMCI/jayeshkrishna/enable_rearr_opts_pio2 +fbfa996 Merge pull request #1269 from ESMCI/jgfouca/code_checker_work_outside_esmci +300cbf3 Merge pull request #1253 from ESMCI/jgfouca/pylint_fix_to_buildnml +d44970c Merge pull request #1249 from ESMCI/jgfouca/create_newcase_non_interactive +8270602 Merge pull request #1231 from ESMCI/mvertens/drv_flds_in +5f3072c Merge pull request #1237 from jedwards4b/jpe_fix_atm_simple_settings +ffa61ea Merge pull request #1242 from jedwards4b/jpe/futurewarning +fecaaad Merge pull request #1240 from ESMCI/jgfouca/fixes_to_submit_phase_pr + + +Modified files: git diff --name-status [previous_tag] +M config/acme/allactive/config_pes.xml +M config/acme/allactive/config_pesall.xml +M config/acme/config_archive.xml +M config/acme/config_files.xml +M config/acme/config_grids.xml +M config/acme/machines/Makefile +M config/acme/machines/config_batch.xml +M config/acme/machines/config_compilers.xml +M config/acme/machines/config_machines.xml +M config/acme/machines/config_pio.xml +A config/acme/machines/syslog.anvil +A config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds +A config/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands +A config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds +M config/cesm/config_files.xml +M config/cesm/config_grids.xml +M config/cesm/machines/config_pio.xml +M config/config_tests.xml +M scripts/Tools/check_input_data +M scripts/create_newcase +M scripts/create_test +M scripts/lib/CIME/BuildTools/configure.py +M scripts/lib/CIME/BuildTools/macrowriterbase.py +M scripts/lib/CIME/SystemTests/dae.py +M scripts/lib/CIME/SystemTests/homme.py +A scripts/lib/CIME/SystemTests/nodefail.py +M scripts/lib/CIME/XML/compilers.py +M scripts/lib/CIME/XML/entry_id.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_pes.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/grids.py +M scripts/lib/CIME/XML/namelist_definition.py +A scripts/lib/CIME/aprun.py +M scripts/lib/CIME/bless_test_results.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/buildnml.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/case_st_archive.py +M scripts/lib/CIME/case_submit.py +M scripts/lib/CIME/code_checker.py +M scripts/lib/CIME/get_timing.py +M scripts/lib/CIME/namelist.py +M scripts/lib/CIME/nmlgen.py +M scripts/lib/CIME/preview_namelists.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/utils.py +M scripts/lib/update_acme_tests.py +M scripts/manage_case +M scripts/tests/scripts_regression_tests.py +M src/build_scripts/buildlib.pio +M src/components/data_comps/datm/cime_config/buildnml +M src/components/data_comps/desp/cime_config/buildnml +M src/components/data_comps/dice/cime_config/buildnml +M src/components/data_comps/dlnd/cime_config/buildnml +M src/components/data_comps/docn/cime_config/buildnml +M src/components/data_comps/drof/cime_config/buildnml +M src/components/data_comps/dwav/cime_config/buildnml +D src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml +D src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml +D src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml +D src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml +D src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in +D src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in +D src/drivers/mct/bld/testdir/env_build.xml +D src/drivers/mct/bld/testdir/env_case.xml +D src/drivers/mct/bld/testdir/env_mach_pes.xml +D src/drivers/mct/bld/testdir/env_run.xml +D src/drivers/mct/bld/testdir/runtest +D src/drivers/mct/bld/testdir/user_nl_cpl +M src/drivers/mct/cime_config/buildnml +M src/drivers/mct/cime_config/config_component.xml +M src/drivers/mct/cime_config/config_component_acme.xml +M src/drivers/mct/cime_config/namelist_definition_drv.xml +A src/drivers/mct/cime_config/namelist_definition_drv_flds.xml +M src/drivers/mct/main/cesm_driver.F90 +M src/drivers/mct/shr/seq_comm_mct.F90 +M src/externals/pio1/pio/CMakeLists.txt +M src/share/esmf_wrf_timemgr/ESMF_Stubs.F90 +M src/share/unit_test_stubs/pio/pio.F90.in +M src/share/util/shr_pio_mod.F90 +M tools/Readme.configure +M tools/configure +M tools/cprnc/summarize_cprnc_diffs +M tools/mapping/map_field/src/map_field.F90 + +====================================================================== + +Originator: Chris Fischer +Date: 3-14-2017 +Tag: cime5.3.0-alpha.04 +Answer Changes: Answer changes for gx3v7 +Tests: scripts_regression_test, SMS.f09_g16.B1850 +Dependencies: + +Brief Summary: + -Need to set RTM to NULL in both cases in order to have this work with fix + needed for upcoming pythonization of rtm and mosart buildnml + -Improve create_test output + -Remove a misleading print statement from tests. + -Better way to handle batch errors. + -Change in location of $SRCROOT/config/allactive to $SRCROOT/cime_config. + -Update so that all relevent grids contain a mask variable. + +User interface changes: New SUBMIT phase! + +PR summary: git log --oneline --first-parent [previous_tag]..master +34a244a Merge pull request #1239 from ESMCI/mvertens/fix_ssptest +867711d Merge pull request #1238 from ESMCI/jgfouca/create_test_better_output +d050c08 Merge pull request #1235 from jedwards4b/mask_updates +5058c33 Merge pull request #1234 from ESMCI/fischer/config_cesm_allactive_mv +64d6bd2 Merge pull request #1230 from ESMCI/jgfouca/add_submit_phase_2 +dec7437 Merge pull request #1232 from jedwards4b/remove_incorrect_statement + + + +Modified files: git diff --name-status [previous_tag] +M config/cesm/config_files.xml +M config/cesm/config_grids.xml +M config/xml_schemas/config_grids_v2.xsd +M scripts/lib/CIME/SystemTests/ssp.py +M scripts/lib/CIME/SystemTests/system_tests_common.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/case_submit.py +M scripts/lib/CIME/case_test.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/test_status.py +M scripts/tests/scripts_regression_tests.p + +====================================================================== +Originator: Chris Fischer +Date: 3-11-2017 +Tag: cime5.3.0-alpha.03 +Answer Changes: None +Tests: scripts_regression_tests, various CESM tests. +Dependencies: + +Brief Summary: + Added new aliases for resolutions supporting a gx1v7 mask. + Move location of config/cesm/allactive. + Get rid of hard coded comp_class settings and use get_values instead. + Fix build model message. + Ignore prereq if forced. + Additional include path needed for cmake check_function_exists. + PIO 2 import. + +User interface changes: + +PR summary: git log --oneline --first-parent [previous_tag]..master + 3627e16 Merge pull request #1228 from jtruesdal/add_gx1v7 + 7b10920 Merge pull request #1227 from ESMCI/fischer/config_cesm_allactive_mv + ab579b3 Merge pull request #1226 from jedwards4b/compclasses_cleanup + dc0a87c Merge pull request #1225 from ESMCI/jgfouca/fix_build_msg + 5f4f989 Merge pull request #1218 from jedwards4b/ignore_prereq_if_forced + ff3d97f Merge pull request #1219 from jedwards4b/cmake_module_fix + 37fe05b Merge pull request #1202 from jedwards4b/pio2_import + +Modified files: git diff --name-status [previous_tag] +M config/acme/machines/Makefile +M config/cesm/config_files.xml +M config/cesm/config_grids.xml +M config/cesm/machines/Makefile +M scripts/lib/CIME/SystemTests/pem.py +M scripts/lib/CIME/SystemTests/pet.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/get_timing.py +M scripts/lib/CIME/test_scheduler.py +M src/build_scripts/buildlib.pio +M src/components/data_comps/docn/cime_config/config_component.xml +M src/externals/pio2/CMakeLists.txt +M src/externals/pio2/CTestConfig.cmake +M src/externals/pio2/CTestScript.cmake +M src/externals/pio2/cmake/FindNetCDF.cmake +M src/externals/pio2/cmake/LibCheck.cmake +M src/externals/pio2/cmake/LibMPI.cmake +A src/externals/pio2/cmake/mpiexec.nwscla +A src/externals/pio2/cmake/mpiexec.olcf +A src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake +M src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +A src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake +M src/externals/pio2/ctest/runcdash-alcf-ibm.sh +A src/externals/pio2/ctest/runcdash-anlworkstation.sh +M src/externals/pio2/ctest/runcdash-cgd-nag.sh +M src/externals/pio2/ctest/runcdash-nersc-cray.sh +M src/externals/pio2/ctest/runcdash-nersc-intel.sh +M src/externals/pio2/ctest/runcdash-nwsc-gnu.sh +M src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +M src/externals/pio2/ctest/runcdash-nwsc-intel.sh +M src/externals/pio2/ctest/runcdash-nwsc-pgi.sh +A src/externals/pio2/ctest/runcdash-nwscla-intel.sh +A src/externals/pio2/ctest/runctest-anlworkstation.sh +M src/externals/pio2/ctest/runctest-cgd.sh +M src/externals/pio2/ctest/runctest-nersc.sh +A src/externals/pio2/ctest/runctest-nwscla.sh +M src/externals/pio2/doc/CMakeLists.txt +M src/externals/pio2/doc/Doxyfile.in +M src/externals/pio2/doc/source/Error.txt +M src/externals/pio2/doc/source/Examples.txt +M src/externals/pio2/doc/source/Installing.txt +A src/externals/pio2/doc/source/Testing.txt +M src/externals/pio2/doc/source/base.txt +A src/externals/pio2/doc/source/contributing_code.txt +M src/externals/pio2/doc/source/mach_walkthrough.txt +M src/externals/pio2/examples/CMakeLists.txt +M src/externals/pio2/examples/c/CMakeLists.txt +M src/externals/pio2/examples/c/example1.c +M src/externals/pio2/examples/c/example2.c +M src/externals/pio2/examples/c/examplePio.c +A src/externals/pio2/examples/c/valsupp_example1.supp +M src/externals/pio2/examples/f03/CMakeLists.txt +M src/externals/pio2/examples/f03/examplePio.f90 +M src/externals/pio2/src/CMakeLists.txt +M src/externals/pio2/src/clib/CMakeLists.txt +M src/externals/pio2/src/clib/bget.c +M src/externals/pio2/src/clib/bget.h +M src/externals/pio2/src/clib/config.h.in +D src/externals/pio2/src/clib/ncparser.pl +D src/externals/pio2/src/clib/ncputgetparser.pl +M src/externals/pio2/src/clib/pio.h +D src/externals/pio2/src/clib/pio_c_get_template.c +D src/externals/pio2/src/clib/pio_c_put_template.c +D src/externals/pio2/src/clib/pio_c_template.c +M src/externals/pio2/src/clib/pio_darray.c +A src/externals/pio2/src/clib/pio_darray_int.c +M src/externals/pio2/src/clib/pio_file.c +M src/externals/pio2/src/clib/pio_get_nc.c +A src/externals/pio2/src/clib/pio_getput_int.c +M src/externals/pio2/src/clib/pio_internal.h +M src/externals/pio2/src/clib/pio_lists.c +A src/externals/pio2/src/clib/pio_msg.c +M src/externals/pio2/src/clib/pio_nc.c +M src/externals/pio2/src/clib/pio_nc4.c +M src/externals/pio2/src/clib/pio_put_nc.c +M src/externals/pio2/src/clib/pio_rearrange.c +M src/externals/pio2/src/clib/pio_spmd.c +A src/externals/pio2/src/clib/pio_varm.c +M src/externals/pio2/src/clib/pioc.c +M src/externals/pio2/src/clib/pioc_sc.c +M src/externals/pio2/src/clib/pioc_support.c +M src/externals/pio2/src/clib/topology.c +M src/externals/pio2/src/flib/CMakeLists.txt +M src/externals/pio2/src/flib/pio.F90 +M src/externals/pio2/src/flib/pio_nf.F90 +M src/externals/pio2/src/flib/pio_types.F90 +M src/externals/pio2/src/flib/piodarray.F90.in +M src/externals/pio2/src/flib/piolib_mod.F90 +M src/externals/pio2/src/flib/pionfatt_mod.F90.in +M src/externals/pio2/src/flib/pionfget_mod.F90.in +M src/externals/pio2/src/flib/pionfput_mod.F90.in +M src/externals/pio2/src/gptl/CMakeLists.txt +M src/externals/pio2/src/gptl/gptl.c +M src/externals/pio2/src/gptl/perf_mod.F90 +M src/externals/pio2/tests/CMakeLists.txt +A src/externals/pio2/tests/cunit/CMakeLists.txt +A src/externals/pio2/tests/cunit/pio_tests.h +A src/externals/pio2/tests/cunit/test_async_2comp.c +A src/externals/pio2/tests/cunit/test_async_3proc.c +A src/externals/pio2/tests/cunit/test_async_4proc.c +A src/externals/pio2/tests/cunit/test_async_simple.c +A src/externals/pio2/tests/cunit/test_common.c +A src/externals/pio2/tests/cunit/test_darray.c +A src/externals/pio2/tests/cunit/test_darray_1d.c +A src/externals/pio2/tests/cunit/test_darray_3d.c +A src/externals/pio2/tests/cunit/test_darray_multivar.c +A src/externals/pio2/tests/cunit/test_decomps.c +A src/externals/pio2/tests/cunit/test_intercomm2.c +A src/externals/pio2/tests/cunit/test_iosystem2.c +A src/externals/pio2/tests/cunit/test_iosystem2_simple.c +A src/externals/pio2/tests/cunit/test_iosystem2_simple2.c +A src/externals/pio2/tests/cunit/test_iosystem3.c +A src/externals/pio2/tests/cunit/test_iosystem3_simple.c +A src/externals/pio2/tests/cunit/test_iosystem3_simple2.c +A src/externals/pio2/tests/cunit/test_pioc.c +A src/externals/pio2/tests/cunit/test_pioc_fill.c +A src/externals/pio2/tests/cunit/test_pioc_putget.c +A src/externals/pio2/tests/cunit/test_pioc_unlim.c +A src/externals/pio2/tests/cunit/test_shared.c +A src/externals/pio2/tests/cunit/test_spmd.c +M src/externals/pio2/tests/general/CMakeLists.txt +A src/externals/pio2/tests/general/README.md +M src/externals/pio2/tests/general/ncdf_fail.F90.in +M src/externals/pio2/tests/general/ncdf_get_put.F90.in +A src/externals/pio2/tests/general/ncdf_inq.F90.in +D src/externals/pio2/tests/general/pio_decomp_fillval2.F90 +A src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in +M src/externals/pio2/tests/general/pio_decomp_tests.F90.in +A src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in +A src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in +A src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in +M src/externals/pio2/tests/general/pio_file_simple_tests.F90.in +A src/externals/pio2/tests/general/pio_iosystem_tests.F90.in +A src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in +A src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in +A src/externals/pio2/tests/general/pio_rearr.F90.in +A src/externals/pio2/tests/general/pio_rearr_opts.F90.in +A src/externals/pio2/tests/general/pio_rearr_opts2.F90.in +M src/externals/pio2/tests/general/test_memleak.c +M src/externals/pio2/tests/general/util/pio_tf_f90gen.pl +M src/externals/pio2/tests/general/util/pio_tutil.F90 +M src/externals/pio2/tests/performance/CMakeLists.txt +M src/externals/pio2/tests/unit/CMakeLists.txt +D src/externals/pio2/tests/unit/README +M src/externals/pio2/tests/unit/basic_tests.F90 +M src/externals/pio2/tests/unit/driver.F90 +M src/externals/pio2/tests/unit/global_vars.F90 +M src/externals/pio2/tests/unit/nc_set_log_level2.c +M src/externals/pio2/tests/unit/ncdf_tests.F90 +D src/externals/pio2/tests/unit/test_names.c +D src/externals/pio2/tests/unit/test_nc4.c + +====================================================================== +Originator: Chris Fischer +Date: 3-7-2017 +Tag: cime5.3.0-alpha.02 +Answer Changes: None +Tests: +Dependencies: + +Brief Summary: Revert PR# 1210, remove hardcoded compclasses + +User interface changes: + +PR summary: git log --oneline --first-parent [previous_tag]..master + 4cc7e22 Merge pull request #1213 from ESMCI/revert-1210-compclasses_cleanup + +Modified files: git diff --name-status [previous_tag] +M scripts/lib/CIME/SystemTests/ncr.py +M scripts/lib/CIME/SystemTests/pem.py +M scripts/lib/CIME/SystemTests/pet.py +M scripts/lib/CIME/get_timing.py + +====================================================================== + +Originator: Chris Fischer +Date: 3-6-2017 +Tag: cime5.3.0-alpha.01 +Answer Changes: None +Tests: code_checker,hist_utils.py,scripts_regression_tests.py, on + different PRs.. +Dependencies: + +Brief Summary: + - Adjust pio layout if pe layout is changed. + - Make baseline perms setting more robust. + - Allows user to rm -rf their build area and rebuild without having + to re-run case.setup. + - Big refactor of CaseStatus management. + - cdf64 only valid for serial netcdf, fix seq test + - Fix bug in ERP namelist comparison. + - Refactor env_batch + - fix issue with compset not printing to timing file + - copy rather than follow links in create_clone + - update compare_xml to find more differences + - remove hardcoded compclasses + +User interface changes: + - case.setup has a new option --no-adjust-pio + - case.build behavior slightly different case.build behavior slightly + different + - CaseStatus will look slightly different + - env_batch.xml now looks different + +PR summary: git log --oneline --first-parent [previous_tag]..master +3fd75db Merge pull request #1210 from jedwards4b/compclasses_cleanup +78cab9a Merge pull request #1207 from jedwards4b/fix_compare_xml +bd0f19d Merge pull request #1209 from jedwards4b/symlinks_in_clone +d308873 Merge pull request #1208 from jedwards4b/fix_compset_not_in_timing +77dc910 Merge pull request #1204 from ESMCI/jgfouca/refactor_env_batch +c10fe46 Merge pull request #1203 from ESMCI/jgfouca/fix_erp_namelist_issue +30e8e79 Merge pull request #1200 from jedwards4b/seq_test_fix +5eee02a Merge pull request #1199 from ESMCI/jgfouca/refactor_case_status +4219bfd Merge pull request #1196 from ESMCI/jgfouca/build_needs_to_make_dirs +33ac01a Merge pull request #1197 from ESMCI/jgfouca/minor_hist_chg +7e7f805 Merge pull request #1195 from jedwards4b/pio_layout_adjust +0131a12 resolve merge conflicts +6c93fc3 Merge branch 'agsalin/mkdir_src' +215e459 Merge pull request #1182 from jedwards4b/config_archive_split + + +Modified files: git diff --name-status [previous_tag] +M CMakeLists.txt +M ChangeLog +D cime_config/acme/allactive/config_compsets.xml +D cime_config/acme/allactive/config_compsets.xml.cime2 +D cime_config/acme/allactive/config_pes.xml +D cime_config/acme/allactive/config_pes.xml.cime2 +D cime_config/acme/allactive/config_pesall.xml +D cime_config/acme/allactive/testlist_allactive.xml +D cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands +D cime_config/acme/allactive/testmods_dirs/force_netcdf_pio/shell_commands +D cime_config/acme/allactive/usermods_dirs/b1850/shell_commands +D cime_config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods +D cime_config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism +D cime_config/acme/config_archive.xml +D cime_config/acme/config_files.xml +D cime_config/acme/config_grids.xml +D cime_config/acme/machines/Depends.cetus +D cime_config/acme/machines/Depends.corip1 +D cime_config/acme/machines/Depends.cray +D cime_config/acme/machines/Depends.gnu +D cime_config/acme/machines/Depends.intel +D cime_config/acme/machines/Depends.intel14 +D cime_config/acme/machines/Depends.intelmic +D cime_config/acme/machines/Depends.intelmic14 +D cime_config/acme/machines/Depends.mira +D cime_config/acme/machines/Depends.nag +D cime_config/acme/machines/Depends.titan.cray +D cime_config/acme/machines/Depends.titan.intel +D cime_config/acme/machines/Depends.titan.pgi +D cime_config/acme/machines/Depends.titan.pgiacc +D cime_config/acme/machines/Makefile +D cime_config/acme/machines/README +D cime_config/acme/machines/config_batch.xml +D cime_config/acme/machines/config_batch.xsd +D cime_config/acme/machines/config_compilers.xml +D cime_config/acme/machines/config_lt_archive.xml +D cime_config/acme/machines/config_machines.xml +D cime_config/acme/machines/config_pio.xml +D cime_config/acme/machines/mkDepends +D cime_config/acme/machines/mkSrcfiles +D cime_config/acme/machines/syslog.cetus +D cime_config/acme/machines/syslog.cori-haswell +D cime_config/acme/machines/syslog.cori-knl +D cime_config/acme/machines/syslog.edison +D cime_config/acme/machines/syslog.mira +D cime_config/acme/machines/syslog.noop +D cime_config/acme/machines/syslog.titan +D cime_config/acme/machines/template.case.run +D cime_config/acme/machines/template.case.test +D cime_config/acme/machines/template.lt_archive +D cime_config/acme/machines/template.ocn_diagnostics +D cime_config/acme/machines/template.st_archive +D cime_config/acme/machines/template.timeseries +D cime_config/acme/machines/userdefined_laptop_template/README.md +D cime_config/acme/machines/userdefined_laptop_template/config_compilers.xml +D cime_config/acme/machines/userdefined_laptop_template/config_machines.xml +D cime_config/acme/machines/userdefined_laptop_template/config_pes.xml +D cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam +D cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm +D cime_config/buildlib.csm_share +D cime_config/buildlib.gptl +D cime_config/buildlib.mct +D cime_config/buildlib.mpi-serial +D cime_config/buildlib.pio +D cime_config/cesm/config_archive.xml +D cime_config/cesm/config_files.xml +D cime_config/cesm/config_grids.xml +D cime_config/cesm/machines/Depends.babbageKnc +D cime_config/cesm/machines/Depends.bluewaters +D cime_config/cesm/machines/Depends.corip1 +D cime_config/cesm/machines/Depends.cray +D cime_config/cesm/machines/Depends.gnu +D cime_config/cesm/machines/Depends.goldbach +D cime_config/cesm/machines/Depends.intel +D cime_config/cesm/machines/Depends.intel14 +D cime_config/cesm/machines/Depends.intelmic +D cime_config/cesm/machines/Depends.intelmic14 +D cime_config/cesm/machines/Depends.mira +D cime_config/cesm/machines/Depends.nag +D cime_config/cesm/machines/Makefile +D cime_config/cesm/machines/README +D cime_config/cesm/machines/config_batch.xml +D cime_config/cesm/machines/config_compilers.xml +D cime_config/cesm/machines/config_lt_archive.xml +D cime_config/cesm/machines/config_machines.xml +D cime_config/cesm/machines/config_pio.xml +D cime_config/cesm/machines/mkDepends +D cime_config/cesm/machines/mkSrcfiles +D cime_config/cesm/machines/nag_mpi_argument.txt +D cime_config/cesm/machines/template.case.run +D cime_config/cesm/machines/template.case.test +D cime_config/cesm/machines/template.lt_archive +D cime_config/cesm/machines/template.st_archive +D cime_config/cesm/machines/userdefined_laptop_template/README.md +D cime_config/cesm/machines/userdefined_laptop_template/config_compilers.xml +D cime_config/cesm/machines/userdefined_laptop_template/config_machines.xml +D cime_config/cesm/machines/userdefined_laptop_template/config_pes.xml +D cime_config/config_headers.xml +D cime_config/config_tests.xml +D cime_config/xml_schemas/archive.xsd +D cime_config/xml_schemas/cimeteststatus.xsd +D cime_config/xml_schemas/config_batch.xsd +D cime_config/xml_schemas/config_compilers_v2.xsd +D cime_config/xml_schemas/config_compsets.xsd +D cime_config/xml_schemas/config_grids_v2.xsd +D cime_config/xml_schemas/config_machines.xsd +D cime_config/xml_schemas/config_machines_template.xml +D cime_config/xml_schemas/config_pes.xsd +D cime_config/xml_schemas/entry_id.xsd +D cime_config/xml_schemas/entry_id_base.xsd +D cime_config/xml_schemas/entry_id_namelist.xsd +D cime_config/xml_schemas/testlist.xsd +D components/data_comps/datm/atm_comp_mct.F90 +D components/data_comps/datm/cime_config/buildlib +D components/data_comps/datm/cime_config/buildnml +D components/data_comps/datm/cime_config/config_component.xml +D components/data_comps/datm/cime_config/namelist_definition_datm.xml +D components/data_comps/datm/cime_config/user_nl_datm +D components/data_comps/datm/datm_comp_mod.F90 +D components/data_comps/datm/datm_shr_mod.F90 +D components/data_comps/datm/doc/SOM.pdf +D components/data_comps/datm/doc/bookinfo.xml +D components/data_comps/datm/doc/datacomps.xml +D components/data_comps/datm/doc/intro.xml +D components/data_comps/datm/doc/rundocbook.csh +D components/data_comps/datm/doc/streams.xml +D components/data_comps/datm/doc/ug.xml +D components/data_comps/desp/cime_config/buildlib +D components/data_comps/desp/cime_config/buildnml +D components/data_comps/desp/cime_config/config_component.xml +D components/data_comps/desp/cime_config/namelist_definition_desp.xml +D components/data_comps/desp/cime_config/user_nl_desp +D components/data_comps/desp/desp_comp_mod.F90 +D components/data_comps/desp/esp_comp_mct.F90 +D components/data_comps/desp/esp_utils.F90 +D components/data_comps/dice/cime_config/buildlib +D components/data_comps/dice/cime_config/buildnml +D components/data_comps/dice/cime_config/config_component.xml +D components/data_comps/dice/cime_config/namelist_definition_dice.xml +D components/data_comps/dice/cime_config/user_nl_dice +D components/data_comps/dice/dice_comp_mod.F90 +D components/data_comps/dice/ice_comp_mct.F90 +D components/data_comps/dlnd/cime_config/buildlib +D components/data_comps/dlnd/cime_config/buildnml +D components/data_comps/dlnd/cime_config/config_component.xml +D components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml +D components/data_comps/dlnd/cime_config/user_nl_dlnd +D components/data_comps/dlnd/dlnd_comp_mod.F90 +D components/data_comps/dlnd/lnd_comp_mct.F90 +D components/data_comps/docn/cime_config/buildlib +D components/data_comps/docn/cime_config/buildnml +D components/data_comps/docn/cime_config/config_component.xml +D components/data_comps/docn/cime_config/namelist_definition_docn.xml +D components/data_comps/docn/cime_config/user_nl_docn +D components/data_comps/docn/docn_comp_mod.F90 +D components/data_comps/docn/ocn_comp_mct.F90 +D components/data_comps/docn/tools/pop_som_frc/README +D components/data_comps/docn/tools/pop_som_frc/SOM.doc +D components/data_comps/docn/tools/pop_som_frc/SOM.pdf +D components/data_comps/docn/tools/pop_som_frc/pop_frc.csh +D components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl +D components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl +D components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl +D components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh +D components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh +D components/data_comps/drof/cime_config/buildlib +D components/data_comps/drof/cime_config/buildnml +D components/data_comps/drof/cime_config/config_component.xml +D components/data_comps/drof/cime_config/namelist_definition_drof.xml +D components/data_comps/drof/cime_config/user_nl_drof +D components/data_comps/drof/drof_comp_mod.F90 +D components/data_comps/drof/rof_comp_mct.F90 +D components/data_comps/dwav/README +D components/data_comps/dwav/bld/README +D components/data_comps/dwav/cime_config/buildlib +D components/data_comps/dwav/cime_config/buildnml +D components/data_comps/dwav/cime_config/config_component.xml +D components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +D components/data_comps/dwav/cime_config/user_nl_dwav +D components/data_comps/dwav/dwav_comp_mod.F90 +D components/data_comps/dwav/wav_comp_mct.F90 +D components/stub_comps/satm/cime_config/buildlib +D components/stub_comps/satm/cime_config/buildnml +D components/stub_comps/satm/cime_config/config_component.xml +D components/stub_comps/satm/cpl/atm_comp_mct.F90 +D components/stub_comps/sesp/cime_config/buildlib +D components/stub_comps/sesp/cime_config/buildnml +D components/stub_comps/sesp/cime_config/config_component.xml +D components/stub_comps/sesp/cpl/esp_comp_mct.F90 +D components/stub_comps/sglc/cime_config/buildlib +D components/stub_comps/sglc/cime_config/buildnml +D components/stub_comps/sglc/cime_config/config_component.xml +D components/stub_comps/sglc/cpl/glc_comp_mct.F90 +D components/stub_comps/sice/cime_config/buildlib +D components/stub_comps/sice/cime_config/buildnml +D components/stub_comps/sice/cime_config/config_component.xml +D components/stub_comps/sice/cpl/ice_comp_mct.F90 +D components/stub_comps/slnd/cime_config/buildlib +D components/stub_comps/slnd/cime_config/buildnml +D components/stub_comps/slnd/cime_config/config_component.xml +D components/stub_comps/slnd/cpl/lnd_comp_mct.F90 +D components/stub_comps/socn/cime_config/buildlib +D components/stub_comps/socn/cime_config/buildnml +D components/stub_comps/socn/cime_config/config_component.xml +D components/stub_comps/socn/cpl/ocn_comp_mct.F90 +D components/stub_comps/srof/cime_config/buildlib +D components/stub_comps/srof/cime_config/buildnml +D components/stub_comps/srof/cime_config/config_component.xml +D components/stub_comps/srof/cpl/rof_comp_mct.F90 +D components/stub_comps/swav/cime_config/buildlib +D components/stub_comps/swav/cime_config/buildnml +D components/stub_comps/swav/cime_config/config_component.xml +D components/stub_comps/swav/cpl/wav_comp_mct.F90 +D components/xcpl_comps/xatm/cime_config/buildlib +D components/xcpl_comps/xatm/cime_config/buildnml +D components/xcpl_comps/xatm/cime_config/config_component.xml +D components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 +D components/xcpl_comps/xglc/cime_config/buildlib +D components/xcpl_comps/xglc/cime_config/buildnml +D components/xcpl_comps/xglc/cime_config/config_component.xml +D components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 +D components/xcpl_comps/xice/cime_config/buildlib +D components/xcpl_comps/xice/cime_config/buildnml +D components/xcpl_comps/xice/cime_config/config_component.xml +D components/xcpl_comps/xice/cpl/ice_comp_mct.F90 +D components/xcpl_comps/xlnd/cime_config/buildlib +D components/xcpl_comps/xlnd/cime_config/buildnml +D components/xcpl_comps/xlnd/cime_config/config_component.xml +D components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 +D components/xcpl_comps/xocn/cime_config/buildlib +D components/xcpl_comps/xocn/cime_config/buildnml +D components/xcpl_comps/xocn/cime_config/config_component.xml +D components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 +D components/xcpl_comps/xrof/cime_config/buildlib +D components/xcpl_comps/xrof/cime_config/buildnml +D components/xcpl_comps/xrof/cime_config/config_component.xml +D components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 +D components/xcpl_comps/xshare/dead_data_mod.F90 +D components/xcpl_comps/xshare/dead_mct_mod.F90 +D components/xcpl_comps/xshare/dead_mod.F90 +D components/xcpl_comps/xwav/cime_config/buildlib +D components/xcpl_comps/xwav/cime_config/buildnml +D components/xcpl_comps/xwav/cime_config/config_component.xml +D components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 +A config/acme/allactive/config_compsets.xml +A config/acme/allactive/config_compsets.xml.cime2 +A config/acme/allactive/config_pes.xml +A config/acme/allactive/config_pes.xml.cime2 +A config/acme/allactive/config_pesall.xml +A config/acme/allactive/testlist_allactive.xml +A config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands +A config/acme/allactive/testmods_dirs/force_netcdf_pio/shell_commands +A config/acme/allactive/usermods_dirs/b1850/shell_commands +A config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods +A config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism +A config/acme/config_archive.xml +A config/acme/config_files.xml +A config/acme/config_grids.xml +A config/acme/machines/Depends.cetus +A config/acme/machines/Depends.corip1 +A config/acme/machines/Depends.cray +A config/acme/machines/Depends.gnu +A config/acme/machines/Depends.intel +A config/acme/machines/Depends.intel14 +A config/acme/machines/Depends.intelmic +A config/acme/machines/Depends.intelmic14 +A config/acme/machines/Depends.mira +A config/acme/machines/Depends.nag +A config/acme/machines/Depends.titan.cray +A config/acme/machines/Depends.titan.intel +A config/acme/machines/Depends.titan.pgi +A config/acme/machines/Depends.titan.pgiacc +A config/acme/machines/Makefile +A config/acme/machines/README +A config/acme/machines/config_batch.xml +A config/acme/machines/config_batch.xsd +A config/acme/machines/config_compilers.xml +A config/acme/machines/config_lt_archive.xml +A config/acme/machines/config_machines.xml +A config/acme/machines/config_pio.xml +A config/acme/machines/mkDepends +A config/acme/machines/mkSrcfiles +A config/acme/machines/syslog.cetus +A config/acme/machines/syslog.cori-haswell +A config/acme/machines/syslog.cori-knl +A config/acme/machines/syslog.edison +A config/acme/machines/syslog.mira +A config/acme/machines/syslog.noop +A config/acme/machines/syslog.titan +A config/acme/machines/template.case.run +A config/acme/machines/template.case.test +A config/acme/machines/template.lt_archive +A config/acme/machines/template.ocn_diagnostics +A config/acme/machines/template.st_archive +A config/acme/machines/template.timeseries +A config/acme/machines/userdefined_laptop_template/README.md +A config/acme/machines/userdefined_laptop_template/config_compilers.xml +A config/acme/machines/userdefined_laptop_template/config_machines.xml +A config/acme/machines/userdefined_laptop_template/config_pes.xml +A config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam +A config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm +A config/cesm/config_archive.xml +A config/cesm/config_files.xml +A config/cesm/config_grids.xml +A config/cesm/machines/Depends.babbageKnc +A config/cesm/machines/Depends.bluewaters +A config/cesm/machines/Depends.corip1 +A config/cesm/machines/Depends.cray +A config/cesm/machines/Depends.gnu +A config/cesm/machines/Depends.goldbach +A config/cesm/machines/Depends.intel +A config/cesm/machines/Depends.intel14 +A config/cesm/machines/Depends.intelmic +A config/cesm/machines/Depends.intelmic14 +A config/cesm/machines/Depends.mira +A config/cesm/machines/Depends.nag +A config/cesm/machines/Makefile +A config/cesm/machines/README +A config/cesm/machines/config_batch.xml +A config/cesm/machines/config_compilers.xml +A config/cesm/machines/config_lt_archive.xml +A config/cesm/machines/config_machines.xml +A config/cesm/machines/config_pio.xml +A config/cesm/machines/mkDepends +A config/cesm/machines/mkSrcfiles +A config/cesm/machines/nag_mpi_argument.txt +A config/cesm/machines/template.case.run +A config/cesm/machines/template.case.test +A config/cesm/machines/template.lt_archive +A config/cesm/machines/template.st_archive +A config/cesm/machines/userdefined_laptop_template/README.md +A config/cesm/machines/userdefined_laptop_template/config_compilers.xml +A config/cesm/machines/userdefined_laptop_template/config_machines.xml +A config/cesm/machines/userdefined_laptop_template/config_pes.xml +A config/config_headers.xml +A config/config_tests.xml +A config/xml_schemas/cimeteststatus.xsd +A config/xml_schemas/config_archive.xsd +A config/xml_schemas/config_batch.xsd +A config/xml_schemas/config_compilers_v2.xsd +A config/xml_schemas/config_compsets.xsd +A config/xml_schemas/config_grids_v2.xsd +A config/xml_schemas/config_machines.xsd +A config/xml_schemas/config_machines_template.xml +A config/xml_schemas/config_pes.xsd +A config/xml_schemas/entry_id.xsd +A config/xml_schemas/entry_id_base.xsd +A config/xml_schemas/entry_id_namelist.xsd +A config/xml_schemas/env_archive.xsd +A config/xml_schemas/env_entry_id.xsd +A config/xml_schemas/testlist.xsd +D driver_cpl/bld/namelist_files/namelist_defaults_drv.xml +D driver_cpl/bld/namelist_files/namelist_definition_drv.xml +D driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml +D driver_cpl/bld/namelist_files/namelist_definition_modio.xml +D driver_cpl/bld/testdir/Buildconf/camconf/drv_flds_in +D driver_cpl/bld/testdir/Buildconf/clmconf/drv_flds_in +D driver_cpl/bld/testdir/env_build.xml +D driver_cpl/bld/testdir/env_case.xml +D driver_cpl/bld/testdir/env_mach_pes.xml +D driver_cpl/bld/testdir/env_run.xml +D driver_cpl/bld/testdir/runtest +D driver_cpl/bld/testdir/user_nl_cpl +D driver_cpl/cime_config/buildexe +D driver_cpl/cime_config/buildnml +D driver_cpl/cime_config/config_component.xml +D driver_cpl/cime_config/config_component_acme.xml +D driver_cpl/cime_config/config_component_cesm.xml +D driver_cpl/cime_config/config_compsets.xml +D driver_cpl/cime_config/config_pes.xml +D driver_cpl/cime_config/namelist_definition_drv.xml +D driver_cpl/cime_config/namelist_definition_modelio.xml +D driver_cpl/cime_config/testdefs/testlist_drv.xml +D driver_cpl/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands +D driver_cpl/cime_config/user_nl_cpl +D driver_cpl/doc/bookinfo.xml +D driver_cpl/doc/chap1.xml +D driver_cpl/doc/chap2.xml +D driver_cpl/doc/figx1.jpg +D driver_cpl/doc/namelist.xml +D driver_cpl/doc/rundocbook.csh +D driver_cpl/doc/stylesheet.dsl +D driver_cpl/doc/ug.xml +D driver_cpl/driver/CMakeLists.txt +D driver_cpl/driver/cesm_comp_mod.F90 +D driver_cpl/driver/cesm_driver.F90 +D driver_cpl/driver/component_mod.F90 +D driver_cpl/driver/component_type_mod.F90 +D driver_cpl/driver/cplcomp_exchange_mod.F90 +D driver_cpl/driver/map_glc2lnd_mod.F90 +D driver_cpl/driver/map_lnd2glc_mod.F90 +D driver_cpl/driver/map_lnd2rof_irrig_mod.F90 +D driver_cpl/driver/mrg_mod.F90 +D driver_cpl/driver/prep_aoflux_mod.F90 +D driver_cpl/driver/prep_atm_mod.F90 +D driver_cpl/driver/prep_glc_mod.F90 +D driver_cpl/driver/prep_ice_mod.F90 +D driver_cpl/driver/prep_lnd_mod.F90 +D driver_cpl/driver/prep_ocn_mod.F90 +D driver_cpl/driver/prep_rof_mod.F90 +D driver_cpl/driver/prep_wav_mod.F90 +D driver_cpl/driver/seq_diag_mct.F90 +D driver_cpl/driver/seq_domain_mct.F90 +D driver_cpl/driver/seq_flux_mct.F90 +D driver_cpl/driver/seq_frac_mct.F90 +D driver_cpl/driver/seq_hist_mod.F90 +D driver_cpl/driver/seq_io_mod.F90 +D driver_cpl/driver/seq_map_mod.F90 +D driver_cpl/driver/seq_map_type_mod.F90 +D driver_cpl/driver/seq_rest_mod.F90 +D driver_cpl/driver/t_driver_timers_mod.F90 +D driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 +D driver_cpl/driver/vertical_gradient_calculator_base.F90 +D driver_cpl/driver/vertical_gradient_calculator_factory.F90 +D driver_cpl/shr/CMakeLists.txt +D driver_cpl/shr/glc_elevclass_mod.F90 +D driver_cpl/shr/seq_cdata_mod.F90 +D driver_cpl/shr/seq_comm_mct.F90 +D driver_cpl/shr/seq_drydep_mod.F90 +D driver_cpl/shr/seq_flds_mod.F90 +D driver_cpl/shr/seq_infodata_mod.F90 +D driver_cpl/shr/seq_io_read_mod.F90 +D driver_cpl/shr/seq_timemgr_mod.F90 +D driver_cpl/shr/shr_carma_mod.F90 +D driver_cpl/shr/shr_expr_parser_mod.F90 +D driver_cpl/shr/shr_fire_emis_mod.F90 +D driver_cpl/shr/shr_megan_mod.F90 +D driver_cpl/unit_test/CMakeLists.txt +D driver_cpl/unit_test/avect_wrapper_test/CMakeLists.txt +D driver_cpl/unit_test/avect_wrapper_test/test_avect_wrapper.pf +D driver_cpl/unit_test/glc_elevclass_test/CMakeLists.txt +D driver_cpl/unit_test/glc_elevclass_test/test_glc_elevclass.pf +D driver_cpl/unit_test/map_glc2lnd_test/CMakeLists.txt +D driver_cpl/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf +D driver_cpl/unit_test/map_lnd2glc_test/CMakeLists.txt +D driver_cpl/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf +D driver_cpl/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt +D driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf +D driver_cpl/unit_test/seq_map_test/CMakeLists.txt +D driver_cpl/unit_test/seq_map_test/test_seq_map.pf +D driver_cpl/unit_test/stubs/CMakeLists.txt +D driver_cpl/unit_test/stubs/vertical_gradient_calculator_constant.F90 +D driver_cpl/unit_test/utils/CMakeLists.txt +D driver_cpl/unit_test/utils/avect_wrapper_mod.F90 +D driver_cpl/unit_test/utils/create_mapper_mod.F90 +D driver_cpl/unit_test/utils/mct_wrapper_mod.F90 +D driver_cpl/unit_test/utils/simple_map_mod.F90 +D driver_cpl/unit_test/vertical_gradient_calculator_test/CMakeLists.txt +D driver_cpl/unit_test/vertical_gradient_calculator_test/README +D driver_cpl/unit_test/vertical_gradient_calculator_test/gradient_example.txt +D driver_cpl/unit_test/vertical_gradient_calculator_test/plot_gradient +D driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf +D driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf +D externals/CMake/.gitignore +D externals/CMake/CESM_utils.cmake +D externals/CMake/CIME_utils.cmake +D externals/CMake/ChangeLog +D externals/CMake/Compilers.cmake +D externals/CMake/FindpFUnit.cmake +D externals/CMake/LICENSE +D externals/CMake/README.md +D externals/CMake/Sourcelist_utils.cmake +D externals/CMake/TryCSizeOf.f90 +D externals/CMake/TryMPIIO.f90 +D externals/CMake/TryMPIMod.f90 +D externals/CMake/TryMPISERIAL.f90 +D externals/CMake/TryPnetcdf_inc.f90 +D externals/CMake/TryPnetcdf_mod.f90 +D externals/CMake/genf90_utils.cmake +D externals/CMake/mpiexec.cmake +D externals/CMake/pFUnit_utils.cmake +D externals/genf90/ChangeLog +D externals/genf90/genf90.pl +D externals/mct/.gitignore +D externals/mct/COPYRIGHT +D externals/mct/Makefile +D externals/mct/Makefile.conf.in +D externals/mct/README +D externals/mct/aclocal.m4 +D externals/mct/benchmarks/.gitignore +D externals/mct/benchmarks/Makefile +D externals/mct/benchmarks/RouterTestDis.F90 +D externals/mct/benchmarks/RouterTestOvr.F90 +D externals/mct/benchmarks/T42.8pC +D externals/mct/benchmarks/T42.8pR +D externals/mct/benchmarks/gx1.8pC +D externals/mct/benchmarks/gx1.8pR +D externals/mct/benchmarks/importBench.F90 +D externals/mct/config.h.in +D externals/mct/configure +D externals/mct/configure.ac +D externals/mct/doc/.gitignore +D externals/mct/doc/Makefile +D externals/mct/doc/README +D externals/mct/doc/coupler.bib +D externals/mct/doc/mct_APIs.tex +D externals/mct/doc/texsrc/.gitignore +D externals/mct/doc/texsrc/Makefile +D externals/mct/doc/texsrc/SRCS_tex.mk +D externals/mct/examples/Makefile +D externals/mct/examples/README +D externals/mct/examples/climate_concur1/.gitignore +D externals/mct/examples/climate_concur1/Makefile +D externals/mct/examples/climate_concur1/README +D externals/mct/examples/climate_concur1/coupler.F90 +D externals/mct/examples/climate_concur1/master.F90 +D externals/mct/examples/climate_concur1/model.F90 +D externals/mct/examples/climate_sequen1/.gitignore +D externals/mct/examples/climate_sequen1/Makefile +D externals/mct/examples/climate_sequen1/README +D externals/mct/examples/climate_sequen1/TS1.dat +D externals/mct/examples/climate_sequen1/coupler.F90 +D externals/mct/examples/climate_sequen1/dst.rc +D externals/mct/examples/climate_sequen1/dstmodel.F90 +D externals/mct/examples/climate_sequen1/master.F90 +D externals/mct/examples/climate_sequen1/mutils.F90 +D externals/mct/examples/climate_sequen1/src.rc +D externals/mct/examples/climate_sequen1/srcmodel.F90 +D externals/mct/examples/simple/.gitignore +D externals/mct/examples/simple/Makefile +D externals/mct/examples/simple/README +D externals/mct/examples/simple/script.babyblue +D externals/mct/examples/simple/twocmp.con.F90 +D externals/mct/examples/simple/twocmp.seq.F90 +D externals/mct/examples/simple/twocmp.seqNB.F90 +D externals/mct/examples/simple/twocmp.seqUnvn.F90 +D externals/mct/install-sh +D externals/mct/m4/README +D externals/mct/m4/acx_mpi.m4 +D externals/mct/m4/ax_fc_version.m4 +D externals/mct/m4/fortran.m4 +D externals/mct/mct/Makefile +D externals/mct/mct/README +D externals/mct/mct/m_Accumulator.F90 +D externals/mct/mct/m_AccumulatorComms.F90 +D externals/mct/mct/m_AttrVect.F90 +D externals/mct/mct/m_AttrVectComms.F90 +D externals/mct/mct/m_AttrVectReduce.F90 +D externals/mct/mct/m_ConvertMaps.F90 +D externals/mct/mct/m_ExchangeMaps.F90 +D externals/mct/mct/m_GeneralGrid.F90 +D externals/mct/mct/m_GeneralGridComms.F90 +D externals/mct/mct/m_GlobalMap.F90 +D externals/mct/mct/m_GlobalSegMap.F90 +D externals/mct/mct/m_GlobalSegMapComms.F90 +D externals/mct/mct/m_GlobalToLocal.F90 +D externals/mct/mct/m_MCTWorld.F90 +D externals/mct/mct/m_MatAttrVectMul.F90 +D externals/mct/mct/m_Merge.F90 +D externals/mct/mct/m_Navigator.F90 +D externals/mct/mct/m_Rearranger.F90 +D externals/mct/mct/m_Router.F90 +D externals/mct/mct/m_SparseMatrix.F90 +D externals/mct/mct/m_SparseMatrixComms.F90 +D externals/mct/mct/m_SparseMatrixDecomp.F90 +D externals/mct/mct/m_SparseMatrixPlus.F90 +D externals/mct/mct/m_SparseMatrixToMaps.F90 +D externals/mct/mct/m_SpatialIntegral.F90 +D externals/mct/mct/m_SpatialIntegralV.F90 +D externals/mct/mct/m_Transfer.F90 +D externals/mct/mkinstalldirs +D externals/mct/mpeu/Makefile +D externals/mct/mpeu/README +D externals/mct/mpeu/assertmpeu.H +D externals/mct/mpeu/get_zeits.c +D externals/mct/mpeu/m_FcComms.F90 +D externals/mct/mpeu/m_FileResolv.F90 +D externals/mct/mpeu/m_Filename.F90 +D externals/mct/mpeu/m_IndexBin_char.F90 +D externals/mct/mpeu/m_IndexBin_integer.F90 +D externals/mct/mpeu/m_IndexBin_logical.F90 +D externals/mct/mpeu/m_List.F90 +D externals/mct/mpeu/m_MergeSorts.F90 +D externals/mct/mpeu/m_Permuter.F90 +D externals/mct/mpeu/m_SortingTools.F90 +D externals/mct/mpeu/m_StrTemplate.F90 +D externals/mct/mpeu/m_String.F90 +D externals/mct/mpeu/m_StringLinkedList.F90 +D externals/mct/mpeu/m_TraceBack.F90 +D externals/mct/mpeu/m_chars.F90 +D externals/mct/mpeu/m_die.F90 +D externals/mct/mpeu/m_dropdead.F90 +D externals/mct/mpeu/m_flow.F90 +D externals/mct/mpeu/m_inpak90.F90 +D externals/mct/mpeu/m_ioutil.F90 +D externals/mct/mpeu/m_mall.F90 +D externals/mct/mpeu/m_mpif.F90 +D externals/mct/mpeu/m_mpif90.F90 +D externals/mct/mpeu/m_mpout.F90 +D externals/mct/mpeu/m_rankMerge.F90 +D externals/mct/mpeu/m_realkinds.F90 +D externals/mct/mpeu/m_stdio.F90 +D externals/mct/mpeu/m_zeit.F90 +D externals/mct/mpi-serial/.gitignore +D externals/mct/mpi-serial/Makefile +D externals/mct/mpi-serial/Makefile.conf.in +D externals/mct/mpi-serial/NOTES +D externals/mct/mpi-serial/README +D externals/mct/mpi-serial/aclocal.m4 +D externals/mct/mpi-serial/cart.c +D externals/mct/mpi-serial/collective.c +D externals/mct/mpi-serial/comm.c +D externals/mct/mpi-serial/config.h.in +D externals/mct/mpi-serial/configure +D externals/mct/mpi-serial/configure.in +D externals/mct/mpi-serial/copy.c +D externals/mct/mpi-serial/fort.F90 +D externals/mct/mpi-serial/getcount.c +D externals/mct/mpi-serial/group.c +D externals/mct/mpi-serial/handles.c +D externals/mct/mpi-serial/info.c +D externals/mct/mpi-serial/list.c +D externals/mct/mpi-serial/list.h +D externals/mct/mpi-serial/listP.h +D externals/mct/mpi-serial/listops.h +D externals/mct/mpi-serial/m4/README +D externals/mct/mpi-serial/m4/ax_fc_version.m4 +D externals/mct/mpi-serial/mpi.c +D externals/mct/mpi-serial/mpi.h +D externals/mct/mpi-serial/mpiP.h +D externals/mct/mpi-serial/mpif.F90 +D externals/mct/mpi-serial/mpif.h +D externals/mct/mpi-serial/op.c +D externals/mct/mpi-serial/pack.c +D externals/mct/mpi-serial/probe.c +D externals/mct/mpi-serial/protify.awk +D externals/mct/mpi-serial/recv.c +D externals/mct/mpi-serial/req.c +D externals/mct/mpi-serial/send.c +D externals/mct/mpi-serial/tests/.gitignore +D externals/mct/mpi-serial/tests/Makefile +D externals/mct/mpi-serial/tests/ctest.c +D externals/mct/mpi-serial/tests/ctest_old.c +D externals/mct/mpi-serial/tests/ftest.F90 +D externals/mct/mpi-serial/tests/ftest_internal.F90 +D externals/mct/mpi-serial/tests/ftest_old.F90 +D externals/mct/mpi-serial/time.c +D externals/mct/mpi-serial/type.c +D externals/mct/mpi-serial/type.h +D externals/mct/mpi-serial/type_const.c +D externals/mct/protex/protex +D externals/mct/testsystem/Makefile +D externals/mct/testsystem/testall/.gitignore +D externals/mct/testsystem/testall/Makefile +D externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 +D externals/mct/testsystem/testall/UNTESTED +D externals/mct/testsystem/testall/ccm.F90 +D externals/mct/testsystem/testall/convertPOPT.F90 +D externals/mct/testsystem/testall/convertgauss.F90 +D externals/mct/testsystem/testall/cpl.F90 +D externals/mct/testsystem/testall/job.ut-all.jaguar +D externals/mct/testsystem/testall/m_ACTEST.F90 +D externals/mct/testsystem/testall/m_AVTEST.F90 +D externals/mct/testsystem/testall/m_GGRIDTEST.F90 +D externals/mct/testsystem/testall/m_GMAPTEST.F90 +D externals/mct/testsystem/testall/m_GSMAPTEST.F90 +D externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 +D externals/mct/testsystem/testall/m_ROUTERTEST.F90 +D externals/mct/testsystem/testall/m_SMATTEST.F90 +D externals/mct/testsystem/testall/master.F90 +D externals/mct/testsystem/testall/mph.F90 +D externals/mct/testsystem/testall/pop.F90 +D externals/mct/testsystem/testall/processors_map.in +D externals/mct/testsystem/testall/script.jag +D externals/mct/testsystem/testall/ut_SparseMatrix.rc +D externals/mct/testunit/.gitignore +D externals/mct/testunit/AttrVect_Test.F90 +D externals/mct/testunit/Makefile +D externals/mct/testunit/master.F90 +D externals/pio1/CMakeLists.txt +D externals/pio1/ChangeLog +D externals/pio1/ChangeLog_template +D externals/pio1/Doxyfile +D externals/pio1/DoxygenLayout.xml +D externals/pio1/SVN_EXTERNAL_DIRECTORIES +D externals/pio1/customdoxygen.css +D externals/pio1/doc/CAMexample.txt +D externals/pio1/doc/Decomp.txt +D externals/pio1/doc/DoxygenLayout.xml +D externals/pio1/doc/Error.txt +D externals/pio1/doc/Examples.txt +D externals/pio1/doc/GettingStarted.txt +D externals/pio1/doc/Installing.txt +D externals/pio1/doc/api.txt +D externals/pio1/doc/base.txt +D externals/pio1/doc/example/errorhandle +D externals/pio1/doc/example/simple-bc +D externals/pio1/doc/example/simple-bc-rearr +D externals/pio1/doc/example/simple-bc-rearr-pe1 +D externals/pio1/doc/example/simple-bc-rearr-pe2 +D externals/pio1/doc/example/simple-dof +D externals/pio1/doc/example/simple-dof-rearr +D externals/pio1/doc/faq.txt +D externals/pio1/doc/footer.html +D externals/pio1/doc/header.html +D externals/pio1/doc/images/baseimage.graffle +D externals/pio1/doc/images/block-cyclic-rearr.eps +D externals/pio1/doc/images/block-cyclic-rearr.graffle +D externals/pio1/doc/images/block-cyclic-rearr.png +D externals/pio1/doc/images/block-cyclic.eps +D externals/pio1/doc/images/block-cyclic.graffle +D externals/pio1/doc/images/block-cyclic.png +D externals/pio1/doc/images/dof-rearr.eps +D externals/pio1/doc/images/dof-rearr.graffle +D externals/pio1/doc/images/dof-rearr.png +D externals/pio1/doc/images/dof.eps +D externals/pio1/doc/images/dof.graffle +D externals/pio1/doc/images/dof.png +D externals/pio1/doc/testpio_example.txt +D externals/pio1/doxygen.sty +D externals/pio1/pio/CMakeLists.txt +D externals/pio1/pio/C_interface_mod.F90 +D externals/pio1/pio/Makefile.conf.in +D externals/pio1/pio/README.config +D externals/pio1/pio/alloc_mod.F90.in +D externals/pio1/pio/box_rearrange.F90.in +D externals/pio1/pio/calcdecomp.F90 +D externals/pio1/pio/calcdisplace_mod.F90 +D externals/pio1/pio/config.h.in +D externals/pio1/pio/dtypes.h +D externals/pio1/pio/fdepends.awk +D externals/pio1/pio/iompi_mod.F90.in +D externals/pio1/pio/ionf_mod.F90 +D externals/pio1/pio/nf_mod.F90 +D externals/pio1/pio/pio.F90 +D externals/pio1/pio/pio_kinds.F90 +D externals/pio1/pio/pio_mpi_utils.F90 +D externals/pio1/pio/pio_msg_callbacks.F90 +D externals/pio1/pio/pio_msg_getput_callbacks.F90.in +D externals/pio1/pio/pio_msg_mod.F90 +D externals/pio1/pio/pio_nf_utils.F90 +D externals/pio1/pio/pio_spmd_utils.F90.in +D externals/pio1/pio/pio_support.F90 +D externals/pio1/pio/pio_types.F90 +D externals/pio1/pio/pio_utils.F90 +D externals/pio1/pio/piodarray.F90.in +D externals/pio1/pio/piolib_mod.F90 +D externals/pio1/pio/pionfatt_mod.F90.in +D externals/pio1/pio/pionfget_mod.F90.in +D externals/pio1/pio/pionfput_mod.F90.in +D externals/pio1/pio/pionfread_mod.F90.in +D externals/pio1/pio/pionfwrite_mod.F90.in +D externals/pio1/pio/piovdc.F90 +D externals/pio1/pio/rearr_options.h +D externals/pio1/pio/rearrange.F90.in +D externals/pio1/pio/topology.c +D externals/pio1/scripts/Utils.pm +D externals/pio1/scripts/config.pl +D externals/pio1/scripts/testpio_yellowstone.pl +D externals/pio1/tests/testpio/CAM05.csh +D externals/pio1/tests/testpio/CMakeLists.txt +D externals/pio1/tests/testpio/MPASA30km.csh +D externals/pio1/tests/testpio/MPASA60km.csh +D externals/pio1/tests/testpio/POPB.csh +D externals/pio1/tests/testpio/POPC.csh +D externals/pio1/tests/testpio/POPD.csh +D externals/pio1/tests/testpio/POPDv0.csh +D externals/pio1/tests/testpio/POPDv1.csh +D externals/pio1/tests/testpio/POPDv2.csh +D externals/pio1/tests/testpio/POPDv3.csh +D externals/pio1/tests/testpio/POPDv4.csh +D externals/pio1/tests/testpio/POPDv5.csh +D externals/pio1/tests/testpio/README.testpio +D externals/pio1/tests/testpio/WRFB.csh +D externals/pio1/tests/testpio/build_defaults.xml +D externals/pio1/tests/testpio/check_mod.F90 +D externals/pio1/tests/testpio/config_bench.xml +D externals/pio1/tests/testpio/fdepends.awk +D externals/pio1/tests/testpio/gdecomp_mod.F90 +D externals/pio1/tests/testpio/kinds_mod.F90 +D externals/pio1/tests/testpio/kraken.128.csh +D externals/pio1/tests/testpio/kraken.1K.csh +D externals/pio1/tests/testpio/kraken.256.csh +D externals/pio1/tests/testpio/kraken.512.csh +D externals/pio1/tests/testpio/kraken.64.csh +D externals/pio1/tests/testpio/namelist_mod.F90 +D externals/pio1/tests/testpio/namelists/testpio_in.apb05 +D externals/pio1/tests/testpio/namelists/testpio_in.asb01 +D externals/pio1/tests/testpio/namelists/testpio_in.asb04 +D externals/pio1/tests/testpio/namelists/testpio_in.b01 +D externals/pio1/tests/testpio/namelists/testpio_in.b02 +D externals/pio1/tests/testpio/namelists/testpio_in.b03 +D externals/pio1/tests/testpio/namelists/testpio_in.b04 +D externals/pio1/tests/testpio/namelists/testpio_in.b05 +D externals/pio1/tests/testpio/namelists/testpio_in.b06 +D externals/pio1/tests/testpio/namelists/testpio_in.b07 +D externals/pio1/tests/testpio/namelists/testpio_in.b08 +D externals/pio1/tests/testpio/namelists/testpio_in.b09 +D externals/pio1/tests/testpio/namelists/testpio_in.b10 +D externals/pio1/tests/testpio/namelists/testpio_in.b11 +D externals/pio1/tests/testpio/namelists/testpio_in.b12 +D externals/pio1/tests/testpio/namelists/testpio_in.b13 +D externals/pio1/tests/testpio/namelists/testpio_in.bb01 +D externals/pio1/tests/testpio/namelists/testpio_in.bb02 +D externals/pio1/tests/testpio/namelists/testpio_in.bb03 +D externals/pio1/tests/testpio/namelists/testpio_in.bb04 +D externals/pio1/tests/testpio/namelists/testpio_in.bb05 +D externals/pio1/tests/testpio/namelists/testpio_in.bb06 +D externals/pio1/tests/testpio/namelists/testpio_in.bb07 +D externals/pio1/tests/testpio/namelists/testpio_in.bb08 +D externals/pio1/tests/testpio/namelists/testpio_in.bn01 +D externals/pio1/tests/testpio/namelists/testpio_in.bn02 +D externals/pio1/tests/testpio/namelists/testpio_in.bn03 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b01 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b02 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b03 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b04 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b05 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b06 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b07 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b08 +D externals/pio1/tests/testpio/namelists/testpio_in.n4n01 +D externals/pio1/tests/testpio/namelists/testpio_in.n4n02 +D externals/pio1/tests/testpio/namelists/testpio_in.n4n03 +D externals/pio1/tests/testpio/namelists/testpio_in.pb01 +D externals/pio1/tests/testpio/namelists/testpio_in.pb02 +D externals/pio1/tests/testpio/namelists/testpio_in.pb03 +D externals/pio1/tests/testpio/namelists/testpio_in.pb04 +D externals/pio1/tests/testpio/namelists/testpio_in.pb05 +D externals/pio1/tests/testpio/namelists/testpio_in.pb06 +D externals/pio1/tests/testpio/namelists/testpio_in.pb07 +D externals/pio1/tests/testpio/namelists/testpio_in.pb08 +D externals/pio1/tests/testpio/namelists/testpio_in.pn01 +D externals/pio1/tests/testpio/namelists/testpio_in.pn02 +D externals/pio1/tests/testpio/namelists/testpio_in.pn03 +D externals/pio1/tests/testpio/namelists/testpio_in.sb01 +D externals/pio1/tests/testpio/namelists/testpio_in.sb02 +D externals/pio1/tests/testpio/namelists/testpio_in.sb03 +D externals/pio1/tests/testpio/namelists/testpio_in.sb04 +D externals/pio1/tests/testpio/namelists/testpio_in.sb05 +D externals/pio1/tests/testpio/namelists/testpio_in.sb06 +D externals/pio1/tests/testpio/namelists/testpio_in.sb07 +D externals/pio1/tests/testpio/namelists/testpio_in.sb08 +D externals/pio1/tests/testpio/namelists/testpio_in.sn01 +D externals/pio1/tests/testpio/namelists/testpio_in.sn02 +D externals/pio1/tests/testpio/namelists/testpio_in.sn03 +D externals/pio1/tests/testpio/namelists/testpio_in.wr01 +D externals/pio1/tests/testpio/namelists/testpio_in.wr02 +D externals/pio1/tests/testpio/perl5lib/ChangeLog +D externals/pio1/tests/testpio/perl5lib/README +D externals/pio1/tests/testpio/perl5lib/XML/Changes +D externals/pio1/tests/testpio/perl5lib/XML/Lite.pm +D externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm +D externals/pio1/tests/testpio/perl5lib/XML/README +D externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 +D externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 +D externals/pio1/tests/testpio/test.csh +D externals/pio1/tests/testpio/test_lib.F90 +D externals/pio1/tests/testpio/testdecomp.F90 +D externals/pio1/tests/testpio/testdecomp.bluefire.run +D externals/pio1/tests/testpio/testdecomp_in +D externals/pio1/tests/testpio/testpio.F90 +D externals/pio1/tests/testpio/testpio_bench.pl +D externals/pio1/tests/testpio/testpio_build.pl +D externals/pio1/tests/testpio/testpio_run.pl +D externals/pio1/tests/testpio/utils_mod.F90 +D externals/pio1/tests/testpio/ystest.sh +D externals/pio1/tests/unittests/CMakeLists.txt +D externals/pio1/tests/unittests/Levy_Notes +D externals/pio1/tests/unittests/README +D externals/pio1/tests/unittests/basic_tests.F90 +D externals/pio1/tests/unittests/driver.F90 +D externals/pio1/tests/unittests/global_vars.F90 +D externals/pio1/tests/unittests/input.nl +D externals/pio1/tests/unittests/nc_set_log_level2.c +D externals/pio1/tests/unittests/ncdf_tests.F90 +D externals/pio1/tests/unittests/not_netcdf.ieee +D externals/pio1/timing/CMakeLists.txt +D externals/pio1/timing/COPYING +D externals/pio1/timing/ChangeLog +D externals/pio1/timing/GPTLget_memusage.c +D externals/pio1/timing/GPTLprint_memusage.c +D externals/pio1/timing/GPTLutil.c +D externals/pio1/timing/Makefile +D externals/pio1/timing/README +D externals/pio1/timing/XXXdotF/perf_mod.F +D externals/pio1/timing/XXXdotF/perf_utils.F +D externals/pio1/timing/f_wrappers.c +D externals/pio1/timing/gptl.c +D externals/pio1/timing/gptl.h +D externals/pio1/timing/gptl.inc +D externals/pio1/timing/gptl_papi.c +D externals/pio1/timing/perf_mod.F90 +D externals/pio1/timing/perf_utils.F90 +D externals/pio1/timing/private.h +D externals/pio1/timing/threadutil.c +D externals/pio2/.gitignore +D externals/pio2/CMakeLists.txt +D externals/pio2/CTestConfig.cmake +D externals/pio2/CTestScript.cmake +D externals/pio2/README.md +D externals/pio2/cmake/FindGPTL.cmake +D externals/pio2/cmake/FindHDF5.cmake +D externals/pio2/cmake/FindLIBRT.cmake +D externals/pio2/cmake/FindLIBZ.cmake +D externals/pio2/cmake/FindMPE.cmake +D externals/pio2/cmake/FindMPISERIAL.cmake +D externals/pio2/cmake/FindNetCDF.cmake +D externals/pio2/cmake/FindPAPI.cmake +D externals/pio2/cmake/FindPnetCDF.cmake +D externals/pio2/cmake/FindSZIP.cmake +D externals/pio2/cmake/LibCheck.cmake +D externals/pio2/cmake/LibFind.cmake +D externals/pio2/cmake/LibMPI.cmake +D externals/pio2/cmake/TryHDF5_HAS_SZIP.c +D externals/pio2/cmake/TryNetCDF_DAP.c +D externals/pio2/cmake/TryNetCDF_PARALLEL.c +D externals/pio2/cmake/TryNetCDF_PNETCDF.c +D externals/pio2/cmake/mpiexec.alcf +D externals/pio2/cmake/mpiexec.ncsa +D externals/pio2/cmake/mpiexec.nersc +D externals/pio2/cmake/mpiexec.nwsc +D externals/pio2/ctest/CTestEnvironment-alcf.cmake +D externals/pio2/ctest/CTestEnvironment-cgd.cmake +D externals/pio2/ctest/CTestEnvironment-ncsa.cmake +D externals/pio2/ctest/CTestEnvironment-nersc.cmake +D externals/pio2/ctest/CTestEnvironment-nwsc.cmake +D externals/pio2/ctest/CTestEnvironment-unknown.cmake +D externals/pio2/ctest/CTestScript-Test.cmake +D externals/pio2/ctest/runcdash-alcf-ibm.sh +D externals/pio2/ctest/runcdash-cgd-nag.sh +D externals/pio2/ctest/runcdash-nersc-cray.sh +D externals/pio2/ctest/runcdash-nersc-intel.sh +D externals/pio2/ctest/runcdash-nwsc-gnu.sh +D externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +D externals/pio2/ctest/runcdash-nwsc-intel.sh +D externals/pio2/ctest/runcdash-nwsc-pgi.sh +D externals/pio2/ctest/runctest-alcf.sh +D externals/pio2/ctest/runctest-cgd.sh +D externals/pio2/ctest/runctest-ncsa.sh +D externals/pio2/ctest/runctest-nersc.sh +D externals/pio2/ctest/runctest-nwsc.sh +D externals/pio2/ctest/runctest-unknown.sh +D externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake +D externals/pio2/doc/CMakeFiles/CMakeOutput.log +D externals/pio2/doc/CMakeFiles/cmake.check_cache +D externals/pio2/doc/CMakeLists.txt +D externals/pio2/doc/Doxyfile.in +D externals/pio2/doc/DoxygenLayout.xml +D externals/pio2/doc/customdoxygen.css +D externals/pio2/doc/doxygen.sty +D externals/pio2/doc/images/baseimage.graffle +D externals/pio2/doc/images/block-cyclic-rearr.eps +D externals/pio2/doc/images/block-cyclic-rearr.graffle +D externals/pio2/doc/images/block-cyclic-rearr.png +D externals/pio2/doc/images/block-cyclic.eps +D externals/pio2/doc/images/block-cyclic.graffle +D externals/pio2/doc/images/block-cyclic.png +D externals/pio2/doc/images/dof-rearr.eps +D externals/pio2/doc/images/dof-rearr.graffle +D externals/pio2/doc/images/dof-rearr.png +D externals/pio2/doc/images/dof.eps +D externals/pio2/doc/images/dof.graffle +D externals/pio2/doc/images/dof.png +D externals/pio2/doc/source/CAMexample.txt +D externals/pio2/doc/source/Decomp.txt +D externals/pio2/doc/source/Error.txt +D externals/pio2/doc/source/Examples.txt +D externals/pio2/doc/source/GettingStarted.txt +D externals/pio2/doc/source/Installing.txt +D externals/pio2/doc/source/api.txt +D externals/pio2/doc/source/base.txt +D externals/pio2/doc/source/example/errorhandle +D externals/pio2/doc/source/example/simple-bc +D externals/pio2/doc/source/example/simple-bc-rearr +D externals/pio2/doc/source/example/simple-bc-rearr-pe1 +D externals/pio2/doc/source/example/simple-bc-rearr-pe2 +D externals/pio2/doc/source/example/simple-dof +D externals/pio2/doc/source/example/simple-dof-rearr +D externals/pio2/doc/source/faq.txt +D externals/pio2/doc/source/mach_walkthrough.txt +D externals/pio2/doc/source/testpio_example.txt +D externals/pio2/examples/CMakeLists.txt +D externals/pio2/examples/basic/CAM05.csh +D externals/pio2/examples/basic/CMakeLists.txt +D externals/pio2/examples/basic/MPASA30km.csh +D externals/pio2/examples/basic/MPASA60km.csh +D externals/pio2/examples/basic/POPB.csh +D externals/pio2/examples/basic/POPC.csh +D externals/pio2/examples/basic/POPD.csh +D externals/pio2/examples/basic/POPDv0.csh +D externals/pio2/examples/basic/POPDv1.csh +D externals/pio2/examples/basic/POPDv2.csh +D externals/pio2/examples/basic/POPDv3.csh +D externals/pio2/examples/basic/POPDv4.csh +D externals/pio2/examples/basic/POPDv5.csh +D externals/pio2/examples/basic/README.testpio +D externals/pio2/examples/basic/WRFB.csh +D externals/pio2/examples/basic/alloc_mod.F90.in +D externals/pio2/examples/basic/build_defaults.xml +D externals/pio2/examples/basic/check_mod.F90 +D externals/pio2/examples/basic/config_bench.xml +D externals/pio2/examples/basic/fdepends.awk +D externals/pio2/examples/basic/gdecomp_mod.F90 +D externals/pio2/examples/basic/kinds_mod.F90 +D externals/pio2/examples/basic/kraken.128.csh +D externals/pio2/examples/basic/kraken.1K.csh +D externals/pio2/examples/basic/kraken.256.csh +D externals/pio2/examples/basic/kraken.512.csh +D externals/pio2/examples/basic/kraken.64.csh +D externals/pio2/examples/basic/namelist_mod.F90 +D externals/pio2/examples/basic/namelists/testpio_in.apb05 +D externals/pio2/examples/basic/namelists/testpio_in.asb01 +D externals/pio2/examples/basic/namelists/testpio_in.asb04 +D externals/pio2/examples/basic/namelists/testpio_in.n4b01 +D externals/pio2/examples/basic/namelists/testpio_in.n4b02 +D externals/pio2/examples/basic/namelists/testpio_in.n4b03 +D externals/pio2/examples/basic/namelists/testpio_in.n4b04 +D externals/pio2/examples/basic/namelists/testpio_in.n4b05 +D externals/pio2/examples/basic/namelists/testpio_in.n4b06 +D externals/pio2/examples/basic/namelists/testpio_in.n4b07 +D externals/pio2/examples/basic/namelists/testpio_in.n4b08 +D externals/pio2/examples/basic/namelists/testpio_in.n4n01 +D externals/pio2/examples/basic/namelists/testpio_in.n4n02 +D externals/pio2/examples/basic/namelists/testpio_in.n4n03 +D externals/pio2/examples/basic/namelists/testpio_in.pb01 +D externals/pio2/examples/basic/namelists/testpio_in.pb02 +D externals/pio2/examples/basic/namelists/testpio_in.pb03 +D externals/pio2/examples/basic/namelists/testpio_in.pb04 +D externals/pio2/examples/basic/namelists/testpio_in.pb05 +D externals/pio2/examples/basic/namelists/testpio_in.pb06 +D externals/pio2/examples/basic/namelists/testpio_in.pb07 +D externals/pio2/examples/basic/namelists/testpio_in.pb08 +D externals/pio2/examples/basic/namelists/testpio_in.pn01 +D externals/pio2/examples/basic/namelists/testpio_in.pn02 +D externals/pio2/examples/basic/namelists/testpio_in.pn03 +D externals/pio2/examples/basic/namelists/testpio_in.ps01 +D externals/pio2/examples/basic/namelists/testpio_in.ps02 +D externals/pio2/examples/basic/namelists/testpio_in.ps03 +D externals/pio2/examples/basic/namelists/testpio_in.ps04 +D externals/pio2/examples/basic/namelists/testpio_in.ps05 +D externals/pio2/examples/basic/namelists/testpio_in.ps06 +D externals/pio2/examples/basic/namelists/testpio_in.ps07 +D externals/pio2/examples/basic/namelists/testpio_in.ps08 +D externals/pio2/examples/basic/namelists/testpio_in.sb01 +D externals/pio2/examples/basic/namelists/testpio_in.sb02 +D externals/pio2/examples/basic/namelists/testpio_in.sb03 +D externals/pio2/examples/basic/namelists/testpio_in.sb04 +D externals/pio2/examples/basic/namelists/testpio_in.sb05 +D externals/pio2/examples/basic/namelists/testpio_in.sb06 +D externals/pio2/examples/basic/namelists/testpio_in.sb07 +D externals/pio2/examples/basic/namelists/testpio_in.sb08 +D externals/pio2/examples/basic/namelists/testpio_in.sn01 +D externals/pio2/examples/basic/namelists/testpio_in.sn02 +D externals/pio2/examples/basic/namelists/testpio_in.sn03 +D externals/pio2/examples/basic/namelists/testpio_in.wr01 +D externals/pio2/examples/basic/namelists/testpio_in.wr02 +D externals/pio2/examples/basic/perl5lib/ChangeLog +D externals/pio2/examples/basic/perl5lib/README +D externals/pio2/examples/basic/perl5lib/XML/Changes +D externals/pio2/examples/basic/perl5lib/XML/Lite.pm +D externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm +D externals/pio2/examples/basic/perl5lib/XML/README +D externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 +D externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 +D externals/pio2/examples/basic/test.csh +D externals/pio2/examples/basic/testdecomp.F90 +D externals/pio2/examples/basic/testdecomp.bluefire.run +D externals/pio2/examples/basic/testdecomp_in +D externals/pio2/examples/basic/testpio.F90 +D externals/pio2/examples/basic/testpio_bench.pl +D externals/pio2/examples/basic/testpio_build.pl +D externals/pio2/examples/basic/testpio_run.pl +D externals/pio2/examples/basic/utils_mod.F90 +D externals/pio2/examples/basic/wstest.c +D externals/pio2/examples/basic/ystest.sh +D externals/pio2/examples/c/CMakeLists.txt +D externals/pio2/examples/c/example1.c +D externals/pio2/examples/c/example2.c +D externals/pio2/examples/c/examplePio.c +D externals/pio2/examples/cxx/CMakeLists.txt +D externals/pio2/examples/cxx/examplePio.cxx +D externals/pio2/examples/f03/CMakeLists.txt +D externals/pio2/examples/f03/examplePio.f90 +D externals/pio2/scripts/prune_decomps.pl +D externals/pio2/src/CMakeLists.txt +D externals/pio2/src/clib/CMakeLists.txt +D externals/pio2/src/clib/bget.c +D externals/pio2/src/clib/bget.h +D externals/pio2/src/clib/config.h.in +D externals/pio2/src/clib/dtypes.h +D externals/pio2/src/clib/ncparser.pl +D externals/pio2/src/clib/ncputgetparser.pl +D externals/pio2/src/clib/pio.h +D externals/pio2/src/clib/pio_c_get_template.c +D externals/pio2/src/clib/pio_c_put_template.c +D externals/pio2/src/clib/pio_c_template.c +D externals/pio2/src/clib/pio_darray.c +D externals/pio2/src/clib/pio_file.c +D externals/pio2/src/clib/pio_get_nc.c +D externals/pio2/src/clib/pio_internal.h +D externals/pio2/src/clib/pio_lists.c +D externals/pio2/src/clib/pio_nc.c +D externals/pio2/src/clib/pio_nc4.c +D externals/pio2/src/clib/pio_put_nc.c +D externals/pio2/src/clib/pio_rearrange.c +D externals/pio2/src/clib/pio_spmd.c +D externals/pio2/src/clib/pioc.c +D externals/pio2/src/clib/pioc_sc.c +D externals/pio2/src/clib/pioc_support.c +D externals/pio2/src/clib/topology.c +D externals/pio2/src/flib/CMakeLists.txt +D externals/pio2/src/flib/pio.F90 +D externals/pio2/src/flib/pio_kinds.F90 +D externals/pio2/src/flib/pio_nf.F90 +D externals/pio2/src/flib/pio_support.F90 +D externals/pio2/src/flib/pio_types.F90 +D externals/pio2/src/flib/piodarray.F90.in +D externals/pio2/src/flib/piolib_mod.F90 +D externals/pio2/src/flib/pionfatt_mod.F90.in +D externals/pio2/src/flib/pionfget_mod.F90.in +D externals/pio2/src/flib/pionfput_mod.F90.in +D externals/pio2/src/gptl/CMakeLists.txt +D externals/pio2/src/gptl/COPYING +D externals/pio2/src/gptl/ChangeLog +D externals/pio2/src/gptl/GPTLget_memusage.c +D externals/pio2/src/gptl/GPTLprint_memusage.c +D externals/pio2/src/gptl/GPTLutil.c +D externals/pio2/src/gptl/README +D externals/pio2/src/gptl/f_wrappers.c +D externals/pio2/src/gptl/gptl.c +D externals/pio2/src/gptl/gptl.h +D externals/pio2/src/gptl/gptl.inc +D externals/pio2/src/gptl/gptl_papi.c +D externals/pio2/src/gptl/perf_mod.F90 +D externals/pio2/src/gptl/perf_utils.F90 +D externals/pio2/src/gptl/private.h +D externals/pio2/src/gptl/threadutil.c +D externals/pio2/tests/CMakeLists.txt +D externals/pio2/tests/general/CMakeLists.txt +D externals/pio2/tests/general/ncdf_fail.F90.in +D externals/pio2/tests/general/ncdf_get_put.F90.in +D externals/pio2/tests/general/ncdf_simple_tests.F90.in +D externals/pio2/tests/general/pio_decomp_fillval.F90.in +D externals/pio2/tests/general/pio_decomp_fillval2.F90 +D externals/pio2/tests/general/pio_decomp_tests.F90.in +D externals/pio2/tests/general/pio_fail.F90.in +D externals/pio2/tests/general/pio_file_fail.F90.in +D externals/pio2/tests/general/pio_file_simple_tests.F90.in +D externals/pio2/tests/general/pio_init_finalize.F90.in +D externals/pio2/tests/general/test_memleak.c +D externals/pio2/tests/general/util/pio_tf_f90gen.pl +D externals/pio2/tests/general/util/pio_tutil.F90 +D externals/pio2/tests/performance/CMakeLists.txt +D externals/pio2/tests/performance/gensimple.pl +D externals/pio2/tests/performance/kt.PIO1.perfmakefile +D externals/pio2/tests/performance/pioperformance.F90 +D externals/pio2/tests/unit/CMakeLists.txt +D externals/pio2/tests/unit/Levy_Notes +D externals/pio2/tests/unit/README +D externals/pio2/tests/unit/basic_tests.F90 +D externals/pio2/tests/unit/driver.F90 +D externals/pio2/tests/unit/global_vars.F90 +D externals/pio2/tests/unit/input.nl +D externals/pio2/tests/unit/nc_set_log_level2.c +D externals/pio2/tests/unit/ncdf_tests.F90 +D externals/pio2/tests/unit/not_netcdf.ieee +D externals/pio2/tests/unit/test_names.c +D externals/pio2/tests/unit/test_nc4.c +D scripts/Testing/Testcases/ICP_script +D scripts/Testing/Testcases/LAR_script +D scripts/Testing/Testcases/OCP_script +D scripts/Testing/Testcases/README +D scripts/Testing/testreporter.pl +M scripts/Tools/case.build +M scripts/Tools/case.setup +M scripts/Tools/standard_script_setup.py +M scripts/Tools/update_acme_tests +M scripts/Tools/xmlchange +M scripts/Tools/xmlconvertors/config_pes_converter.py +M scripts/Tools/xmlconvertors/grid_xml_converter.py +M scripts/Tools/xmlquery +M scripts/create_newcase +A scripts/data_assimilation/da_no_data_mod.sh +A scripts/lib/CIME/BuildTools/__init__.py +A scripts/lib/CIME/BuildTools/cmakemacroswriter.py +A scripts/lib/CIME/BuildTools/configure.py +A scripts/lib/CIME/BuildTools/macroconditiontree.py +A scripts/lib/CIME/BuildTools/macrowriterbase.py +A scripts/lib/CIME/BuildTools/makemacroswriter.py +A scripts/lib/CIME/BuildTools/possiblevalues.py +A scripts/lib/CIME/BuildTools/valuesetting.py +A scripts/lib/CIME/SystemTests/README +A scripts/lib/CIME/SystemTests/__init__.py +A scripts/lib/CIME/SystemTests/dae.py +A scripts/lib/CIME/SystemTests/eri.py +A scripts/lib/CIME/SystemTests/erio.py +A scripts/lib/CIME/SystemTests/erp.py +A scripts/lib/CIME/SystemTests/err.py +A scripts/lib/CIME/SystemTests/ers.py +A scripts/lib/CIME/SystemTests/ert.py +A scripts/lib/CIME/SystemTests/homme.py +A scripts/lib/CIME/SystemTests/icp.py +A scripts/lib/CIME/SystemTests/lii.py +A scripts/lib/CIME/SystemTests/nck.py +A scripts/lib/CIME/SystemTests/ncr.py +A scripts/lib/CIME/SystemTests/pea.py +A scripts/lib/CIME/SystemTests/pem.py +A scripts/lib/CIME/SystemTests/pet.py +A scripts/lib/CIME/SystemTests/pfs.py +A scripts/lib/CIME/SystemTests/pre.py +A scripts/lib/CIME/SystemTests/rep.py +A scripts/lib/CIME/SystemTests/seq.py +A scripts/lib/CIME/SystemTests/sms.py +A scripts/lib/CIME/SystemTests/ssp.py +A scripts/lib/CIME/SystemTests/system_tests_common.py +A scripts/lib/CIME/SystemTests/system_tests_compare_two.py +A scripts/lib/CIME/SystemTests/test_utils/__init__.py +A scripts/lib/CIME/SystemTests/test_utils/user_nl_utils.py +A scripts/lib/CIME/XML/__init__.py +A scripts/lib/CIME/XML/archive.py +A scripts/lib/CIME/XML/batch.py +A scripts/lib/CIME/XML/compilerblock.py +A scripts/lib/CIME/XML/compilers.py +A scripts/lib/CIME/XML/component.py +A scripts/lib/CIME/XML/compsets.py +A scripts/lib/CIME/XML/entry_id.py +A scripts/lib/CIME/XML/env_archive.py +A scripts/lib/CIME/XML/env_base.py +A scripts/lib/CIME/XML/env_batch.py +A scripts/lib/CIME/XML/env_build.py +A scripts/lib/CIME/XML/env_case.py +A scripts/lib/CIME/XML/env_mach_pes.py +A scripts/lib/CIME/XML/env_mach_specific.py +A scripts/lib/CIME/XML/env_run.py +A scripts/lib/CIME/XML/env_test.py +A scripts/lib/CIME/XML/files.py +A scripts/lib/CIME/XML/generic_xml.py +A scripts/lib/CIME/XML/grids.py +A scripts/lib/CIME/XML/headers.py +A scripts/lib/CIME/XML/lt_archive.py +A scripts/lib/CIME/XML/machines.py +A scripts/lib/CIME/XML/namelist_definition.py +A scripts/lib/CIME/XML/pes.py +A scripts/lib/CIME/XML/pio.py +A scripts/lib/CIME/XML/standard_module_setup.py +A scripts/lib/CIME/XML/testlist.py +A scripts/lib/CIME/XML/tests.py +A scripts/lib/CIME/XML/testspec.py +A scripts/lib/CIME/__init__.py +A scripts/lib/CIME/bless_test_results.py +A scripts/lib/CIME/build.py +A scripts/lib/CIME/buildlib.py +A scripts/lib/CIME/buildnml.py +A scripts/lib/CIME/case.py +A scripts/lib/CIME/case_cmpgen_namelists.py +A scripts/lib/CIME/case_lt_archive.py +A scripts/lib/CIME/case_run.py +A scripts/lib/CIME/case_setup.py +A scripts/lib/CIME/case_st_archive.py +A scripts/lib/CIME/case_submit.py +A scripts/lib/CIME/case_test.py +A scripts/lib/CIME/check_input_data.py +A scripts/lib/CIME/check_lockedfiles.py +A scripts/lib/CIME/code_checker.py +A scripts/lib/CIME/compare_namelists.py +A scripts/lib/CIME/compare_test_results.py +A scripts/lib/CIME/get_timing.py +A scripts/lib/CIME/hist_utils.py +A scripts/lib/CIME/namelist.py +A scripts/lib/CIME/nmlgen.py +A scripts/lib/CIME/preview_namelists.py +A scripts/lib/CIME/provenance.py +A scripts/lib/CIME/simple_compare.py +A scripts/lib/CIME/test_scheduler.py +A scripts/lib/CIME/test_status.py +A scripts/lib/CIME/test_utils.py +A scripts/lib/CIME/tests/README +A scripts/lib/CIME/tests/SystemTests/__init__.py +A scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py +A scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py +A scripts/lib/CIME/tests/SystemTests/test_utils/__init__.py +A scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py +A scripts/lib/CIME/tests/__init__.py +A scripts/lib/CIME/tests/case_fake.py +A scripts/lib/CIME/tests/test_case_fake.py +A scripts/lib/CIME/user_mod_support.py +A scripts/lib/CIME/utils.py +A scripts/lib/CIME/wait_for_tests.py +A scripts/lib/cs.status.template +A scripts/lib/cs.submit.template +A scripts/lib/jenkins_generic_job.py +A scripts/lib/update_acme_tests.py +M scripts/manage_testlists +A scripts/tests/CMakeLists.txt +A scripts/tests/CTestConfig.cmake +A scripts/tests/cpl.hi1.nc.test +A scripts/tests/cpl.hi2.nc.test +A scripts/tests/cpl.log.failmemleak.gz +A scripts/tests/cpl.log.passmemleak.gz +A scripts/tests/list_tests +A scripts/tests/scripts_regression_tests.py +A scripts/tests/user_mods_test1/include_user_mods +A scripts/tests/user_mods_test1/user_nl_comp +A scripts/tests/user_mods_test1/user_nl_cpl +A scripts/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 +A scripts/tests/user_mods_test2/user_nl_cpl +D share/csm_share/README +D share/csm_share/include/dynamic_vector_procdef.inc +D share/csm_share/include/dynamic_vector_typedef.inc +D share/csm_share/include/shr_assert.h +D share/csm_share/shr/CMakeLists.txt +D share/csm_share/shr/mct_mod.F90 +D share/csm_share/shr/shr_assert_mod.F90.in +D share/csm_share/shr/shr_cal_mod.F90 +D share/csm_share/shr/shr_const_mod.F90 +D share/csm_share/shr/shr_dmodel_mod.F90 +D share/csm_share/shr/shr_file_mod.F90 +D share/csm_share/shr/shr_flux_mod.F90 +D share/csm_share/shr/shr_frz_mod.F90.in +D share/csm_share/shr/shr_infnan_mod.F90.in +D share/csm_share/shr/shr_kind_mod.F90 +D share/csm_share/shr/shr_log_mod.F90 +D share/csm_share/shr/shr_map_mod.F90 +D share/csm_share/shr/shr_mct_mod.F90 +D share/csm_share/shr/shr_mem_mod.F90 +D share/csm_share/shr/shr_mpi_mod.F90 +D share/csm_share/shr/shr_msg_mod.F90 +D share/csm_share/shr/shr_ncread_mod.F90 +D share/csm_share/shr/shr_nl_mod.F90 +D share/csm_share/shr/shr_orb_mod.F90 +D share/csm_share/shr/shr_pcdf_mod.F90 +D share/csm_share/shr/shr_pio_mod.F90 +D share/csm_share/shr/shr_precip_mod.F90 +D share/csm_share/shr/shr_reprosum_mod.F90 +D share/csm_share/shr/shr_reprosumx86.c +D share/csm_share/shr/shr_scam_mod.F90 +D share/csm_share/shr/shr_spfn_mod.F90 +D share/csm_share/shr/shr_strconvert_mod.F90 +D share/csm_share/shr/shr_strdata_mod.F90 +D share/csm_share/shr/shr_stream_mod.F90 +D share/csm_share/shr/shr_string_mod.F90 +D share/csm_share/shr/shr_sys_mod.F90 +D share/csm_share/shr/shr_tInterp_mod.F90 +D share/csm_share/shr/shr_timer_mod.F90 +D share/csm_share/shr/shr_vmath_mod.F90 +D share/csm_share/shr/shr_wv_sat_mod.F90 +D share/csm_share/shr/water_isotopes.F90 +D share/csm_share/shr/water_types.F90 +D share/csm_share/test/old_unit_testers/Makefile +D share/csm_share/test/old_unit_testers/Mkdepends +D share/csm_share/test/old_unit_testers/Mksrcfiles +D share/csm_share/test/old_unit_testers/bundle_expected.F90 +D share/csm_share/test/old_unit_testers/config.h +D share/csm_share/test/old_unit_testers/make.Macros +D share/csm_share/test/old_unit_testers/namelist +D share/csm_share/test/old_unit_testers/nl/atm.stdin +D share/csm_share/test/old_unit_testers/nl/cpl.stdin +D share/csm_share/test/old_unit_testers/nl/ice.stdin +D share/csm_share/test/old_unit_testers/nl/lnd.stdin +D share/csm_share/test/old_unit_testers/nl/ocn.stdin +D share/csm_share/test/old_unit_testers/run_dshr_bundle_test +D share/csm_share/test/old_unit_testers/run_file_test +D share/csm_share/test/old_unit_testers/test_mod.F90 +D share/csm_share/test/old_unit_testers/test_shr_file.F90 +D share/csm_share/test/old_unit_testers/test_shr_log.F90 +D share/csm_share/test/old_unit_testers/test_shr_mpi.F90 +D share/csm_share/test/old_unit_testers/test_shr_orb.F90 +D share/csm_share/test/old_unit_testers/test_shr_scam.F90 +D share/csm_share/test/old_unit_testers/test_shr_streams.F90 +D share/csm_share/test/old_unit_testers/test_shr_sys.F90 +D share/csm_share/test/old_unit_testers/test_shr_tInterp.F90 +D share/csm_share/test/unit/CMakeLists.txt +D share/csm_share/test/unit/dynamic_vector/CMakeLists.txt +D share/csm_share/test/unit/dynamic_vector/character16_vector_tests.pf.in +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_character16.F90 +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_integer.F90 +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_r8.F90 +D share/csm_share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in +D share/csm_share/test/unit/dynamic_vector/integer_vector_tests.pf.in +D share/csm_share/test/unit/dynamic_vector/ptr_wrapper.F90 +D share/csm_share/test/unit/dynamic_vector/r8_vector_tests.pf.in +D share/csm_share/test/unit/mock/CMakeLists.txt +D share/csm_share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 +D share/csm_share/test/unit/shr_assert_test/CMakeLists.txt +D share/csm_share/test/unit/shr_assert_test/test_assert.pf +D share/csm_share/test/unit/shr_assert_test/test_assert_array.pf +D share/csm_share/test/unit/shr_assert_test/test_macro.pf +D share/csm_share/test/unit/shr_assert_test/test_ndebug.pf +D share/csm_share/test/unit/shr_infnan_test/CMakeLists.txt +D share/csm_share/test/unit/shr_infnan_test/test_infnan.F90 +D share/csm_share/test/unit/shr_log_test/CMakeLists.txt +D share/csm_share/test/unit/shr_log_test/test_error_printers.pf +D share/csm_share/test/unit/shr_precip_test/CMakeLists.txt +D share/csm_share/test/unit/shr_precip_test/test_shr_precip.pf +D share/csm_share/test/unit/shr_spfn_test/CMakeLists.txt +D share/csm_share/test/unit/shr_spfn_test/test_erf_r4.pf +D share/csm_share/test/unit/shr_spfn_test/test_erf_r8.pf +D share/csm_share/test/unit/shr_spfn_test/test_gamma_factorial.pf +D share/csm_share/test/unit/shr_spfn_test/test_igamma.pf +D share/csm_share/test/unit/shr_strconvert_test/CMakeLists.txt +D share/csm_share/test/unit/shr_strconvert_test/test_toString.pf +D share/csm_share/test/unit/shr_string_test/CMakeLists.txt +D share/csm_share/test/unit/shr_string_test/test_shr_string.pf +D share/csm_share/test/unit/shr_vmath_test/CMakeLists.txt +D share/csm_share/test/unit/shr_vmath_test/test_vmath.F90 +D share/csm_share/test/unit/shr_wv_sat_test/CMakeLists.txt +D share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat.pf +D share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf +D share/csm_share/unit_test_stubs/README +D share/csm_share/unit_test_stubs/pio/CMakeLists.txt +D share/csm_share/unit_test_stubs/pio/README +D share/csm_share/unit_test_stubs/pio/pio.F90.in +D share/esmf_wrf_timemgr/CMakeLists.txt +D share/esmf_wrf_timemgr/ESMF.F90 +D share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 +D share/esmf_wrf_timemgr/ESMF_AlarmMod.F90 +D share/esmf_wrf_timemgr/ESMF_BaseMod.F90 +D share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 +D share/esmf_wrf_timemgr/ESMF_CalendarMod.F90 +D share/esmf_wrf_timemgr/ESMF_ClockMod.F90 +D share/esmf_wrf_timemgr/ESMF_FractionMod.F90 +D share/esmf_wrf_timemgr/ESMF_Macros.inc +D share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 +D share/esmf_wrf_timemgr/ESMF_Stubs.F90 +D share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 +D share/esmf_wrf_timemgr/ESMF_TimeMgr.inc +D share/esmf_wrf_timemgr/ESMF_TimeMod.F90 +D share/esmf_wrf_timemgr/Makefile +D share/esmf_wrf_timemgr/MeatMod.F90 +D share/esmf_wrf_timemgr/README +D share/esmf_wrf_timemgr/unittests/Makefile +D share/esmf_wrf_timemgr/unittests/go.csh +D share/esmf_wrf_timemgr/unittests/test.F90 +D share/esmf_wrf_timemgr/unittests/wrf_stuff.F90 +D share/esmf_wrf_timemgr/wrf_error_fatal.F90 +D share/esmf_wrf_timemgr/wrf_message.F90 +D share/shr_RandNum/include/dSFMT-common.h +D share/shr_RandNum/include/dSFMT-params.h +D share/shr_RandNum/include/dSFMT-params19937.h +D share/shr_RandNum/include/dSFMT.h +D share/shr_RandNum/src/dsfmt_f03/dSFMT.c +D share/shr_RandNum/src/dsfmt_f03/dSFMT_interface.F90 +D share/shr_RandNum/src/dsfmt_f03/dSFMT_utils.c +D share/shr_RandNum/src/kissvec/kissvec.c +D share/shr_RandNum/src/kissvec/kissvec_mod.F90 +D share/shr_RandNum/src/mt19937/mersennetwister_mod.F90 +D share/shr_RandNum/src/shr_RandNum_mod.F90 +D share/shr_RandNum/test/bench/Makefile +D share/shr_RandNum/test/bench/test_shr_RandNum.F90 +D share/timing/CMakeLists.txt +D share/timing/COPYING +D share/timing/ChangeLog +D share/timing/GPTLget_memusage.c +D share/timing/GPTLprint_memusage.c +D share/timing/GPTLutil.c +D share/timing/Makefile +D share/timing/README +D share/timing/f_wrappers.c +D share/timing/gptl.c +D share/timing/gptl.h +D share/timing/gptl.inc +D share/timing/gptl_papi.c +D share/timing/perf_mod.F90 +D share/timing/perf_utils.F90 +D share/timing/private.h +A src/build_scripts/buildlib.csm_share +A src/build_scripts/buildlib.gptl +A src/build_scripts/buildlib.mct +A src/build_scripts/buildlib.mpi-serial +A src/build_scripts/buildlib.pio +A src/components/data_comps/datm/atm_comp_mct.F90 +A src/components/data_comps/datm/cime_config/buildlib +A src/components/data_comps/datm/cime_config/buildnml +A src/components/data_comps/datm/cime_config/config_archive.xml +A src/components/data_comps/datm/cime_config/config_component.xml +A src/components/data_comps/datm/cime_config/namelist_definition_datm.xml +A src/components/data_comps/datm/cime_config/user_nl_datm +A src/components/data_comps/datm/datm_comp_mod.F90 +A src/components/data_comps/datm/datm_shr_mod.F90 +A src/components/data_comps/datm/doc/SOM.pdf +A src/components/data_comps/datm/doc/bookinfo.xml +A src/components/data_comps/datm/doc/datacomps.xml +A src/components/data_comps/datm/doc/intro.xml +A src/components/data_comps/datm/doc/rundocbook.csh +A src/components/data_comps/datm/doc/streams.xml +A src/components/data_comps/datm/doc/ug.xml +A src/components/data_comps/desp/cime_config/buildlib +A src/components/data_comps/desp/cime_config/buildnml +A src/components/data_comps/desp/cime_config/config_component.xml +A src/components/data_comps/desp/cime_config/namelist_definition_desp.xml +A src/components/data_comps/desp/cime_config/user_nl_desp +A src/components/data_comps/desp/desp_comp_mod.F90 +A src/components/data_comps/desp/esp_comp_mct.F90 +A src/components/data_comps/desp/esp_utils.F90 +A src/components/data_comps/dice/cime_config/buildlib +A src/components/data_comps/dice/cime_config/buildnml +A src/components/data_comps/dice/cime_config/config_archive.xml +A src/components/data_comps/dice/cime_config/config_component.xml +A src/components/data_comps/dice/cime_config/namelist_definition_dice.xml +A src/components/data_comps/dice/cime_config/user_nl_dice +A src/components/data_comps/dice/dice_comp_mod.F90 +A src/components/data_comps/dice/ice_comp_mct.F90 +A src/components/data_comps/dlnd/cime_config/buildlib +A src/components/data_comps/dlnd/cime_config/buildnml +A src/components/data_comps/dlnd/cime_config/config_archive.xml +A src/components/data_comps/dlnd/cime_config/config_component.xml +A src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml +A src/components/data_comps/dlnd/cime_config/user_nl_dlnd +A src/components/data_comps/dlnd/dlnd_comp_mod.F90 +A src/components/data_comps/dlnd/lnd_comp_mct.F90 +A src/components/data_comps/docn/cime_config/buildlib +A src/components/data_comps/docn/cime_config/buildnml +A src/components/data_comps/docn/cime_config/config_archive.xml +A src/components/data_comps/docn/cime_config/config_component.xml +A src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +A src/components/data_comps/docn/cime_config/user_nl_docn +A src/components/data_comps/docn/docn_comp_mod.F90 +A src/components/data_comps/docn/ocn_comp_mct.F90 +A src/components/data_comps/docn/tools/pop_som_frc/README +A src/components/data_comps/docn/tools/pop_som_frc/SOM.doc +A src/components/data_comps/docn/tools/pop_som_frc/SOM.pdf +A src/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh +A src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl +A src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl +A src/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl +A src/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh +A src/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh +A src/components/data_comps/drof/cime_config/buildlib +A src/components/data_comps/drof/cime_config/buildnml +A src/components/data_comps/drof/cime_config/config_archive.xml +A src/components/data_comps/drof/cime_config/config_component.xml +A src/components/data_comps/drof/cime_config/namelist_definition_drof.xml +A src/components/data_comps/drof/cime_config/user_nl_drof +A src/components/data_comps/drof/drof_comp_mod.F90 +A src/components/data_comps/drof/rof_comp_mct.F90 +A src/components/data_comps/dwav/README +A src/components/data_comps/dwav/bld/README +A src/components/data_comps/dwav/cime_config/buildlib +A src/components/data_comps/dwav/cime_config/buildnml +A src/components/data_comps/dwav/cime_config/config_archive.xml +A src/components/data_comps/dwav/cime_config/config_component.xml +A src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +A src/components/data_comps/dwav/cime_config/user_nl_dwav +A src/components/data_comps/dwav/dwav_comp_mod.F90 +A src/components/data_comps/dwav/wav_comp_mct.F90 +A src/components/stub_comps/satm/cime_config/buildlib +A src/components/stub_comps/satm/cime_config/buildnml +A src/components/stub_comps/satm/cime_config/config_component.xml +A src/components/stub_comps/satm/cpl/atm_comp_mct.F90 +A src/components/stub_comps/sesp/cime_config/buildlib +A src/components/stub_comps/sesp/cime_config/buildnml +A src/components/stub_comps/sesp/cime_config/config_component.xml +A src/components/stub_comps/sesp/cpl/esp_comp_mct.F90 +A src/components/stub_comps/sglc/cime_config/buildlib +A src/components/stub_comps/sglc/cime_config/buildnml +A src/components/stub_comps/sglc/cime_config/config_component.xml +A src/components/stub_comps/sglc/cpl/glc_comp_mct.F90 +A src/components/stub_comps/sice/cime_config/buildlib +A src/components/stub_comps/sice/cime_config/buildnml +A src/components/stub_comps/sice/cime_config/config_component.xml +A src/components/stub_comps/sice/cpl/ice_comp_mct.F90 +A src/components/stub_comps/slnd/cime_config/buildlib +A src/components/stub_comps/slnd/cime_config/buildnml +A src/components/stub_comps/slnd/cime_config/config_component.xml +A src/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 +A src/components/stub_comps/socn/cime_config/buildlib +A src/components/stub_comps/socn/cime_config/buildnml +A src/components/stub_comps/socn/cime_config/config_component.xml +A src/components/stub_comps/socn/cpl/ocn_comp_mct.F90 +A src/components/stub_comps/srof/cime_config/buildlib +A src/components/stub_comps/srof/cime_config/buildnml +A src/components/stub_comps/srof/cime_config/config_component.xml +A src/components/stub_comps/srof/cpl/rof_comp_mct.F90 +A src/components/stub_comps/swav/cime_config/buildlib +A src/components/stub_comps/swav/cime_config/buildnml +A src/components/stub_comps/swav/cime_config/config_component.xml +A src/components/stub_comps/swav/cpl/wav_comp_mct.F90 +A src/components/xcpl_comps/xatm/cime_config/buildlib +A src/components/xcpl_comps/xatm/cime_config/buildnml +A src/components/xcpl_comps/xatm/cime_config/config_component.xml +A src/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 +A src/components/xcpl_comps/xglc/cime_config/buildlib +A src/components/xcpl_comps/xglc/cime_config/buildnml +A src/components/xcpl_comps/xglc/cime_config/config_component.xml +A src/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 +A src/components/xcpl_comps/xice/cime_config/buildlib +A src/components/xcpl_comps/xice/cime_config/buildnml +A src/components/xcpl_comps/xice/cime_config/config_component.xml +A src/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 +A src/components/xcpl_comps/xlnd/cime_config/buildlib +A src/components/xcpl_comps/xlnd/cime_config/buildnml +A src/components/xcpl_comps/xlnd/cime_config/config_component.xml +A src/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 +A src/components/xcpl_comps/xocn/cime_config/buildlib +A src/components/xcpl_comps/xocn/cime_config/buildnml +A src/components/xcpl_comps/xocn/cime_config/config_component.xml +A src/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 +A src/components/xcpl_comps/xrof/cime_config/buildlib +A src/components/xcpl_comps/xrof/cime_config/buildnml +A src/components/xcpl_comps/xrof/cime_config/config_component.xml +A src/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 +A src/components/xcpl_comps/xshare/dead_data_mod.F90 +A src/components/xcpl_comps/xshare/dead_mct_mod.F90 +A src/components/xcpl_comps/xshare/dead_mod.F90 +A src/components/xcpl_comps/xwav/cime_config/buildlib +A src/components/xcpl_comps/xwav/cime_config/buildnml +A src/components/xcpl_comps/xwav/cime_config/config_component.xml +A src/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 +A src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml +A src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml +A src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml +A src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml +A src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in +A src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in +A src/drivers/mct/bld/testdir/env_build.xml +A src/drivers/mct/bld/testdir/env_case.xml +A src/drivers/mct/bld/testdir/env_mach_pes.xml +A src/drivers/mct/bld/testdir/env_run.xml +A src/drivers/mct/bld/testdir/runtest +A src/drivers/mct/bld/testdir/user_nl_cpl +A src/drivers/mct/cime_config/buildexe +A src/drivers/mct/cime_config/buildnml +A src/drivers/mct/cime_config/config_archive.xml +A src/drivers/mct/cime_config/config_component.xml +A src/drivers/mct/cime_config/config_component_acme.xml +A src/drivers/mct/cime_config/config_component_cesm.xml +A src/drivers/mct/cime_config/config_compsets.xml +A src/drivers/mct/cime_config/config_pes.xml +A src/drivers/mct/cime_config/namelist_definition_drv.xml +A src/drivers/mct/cime_config/namelist_definition_modelio.xml +A src/drivers/mct/cime_config/testdefs/testlist_drv.xml +A src/drivers/mct/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands +A src/drivers/mct/cime_config/user_nl_cpl +A src/drivers/mct/doc/bookinfo.xml +A src/drivers/mct/doc/chap1.xml +A src/drivers/mct/doc/chap2.xml +A src/drivers/mct/doc/figx1.jpg +A src/drivers/mct/doc/namelist.xml +A src/drivers/mct/doc/rundocbook.csh +A src/drivers/mct/doc/stylesheet.dsl +A src/drivers/mct/doc/ug.xml +A src/drivers/mct/main/CMakeLists.txt +A src/drivers/mct/main/cesm_comp_mod.F90 +A src/drivers/mct/main/cesm_driver.F90 +A src/drivers/mct/main/component_mod.F90 +A src/drivers/mct/main/component_type_mod.F90 +A src/drivers/mct/main/cplcomp_exchange_mod.F90 +A src/drivers/mct/main/map_glc2lnd_mod.F90 +A src/drivers/mct/main/map_lnd2glc_mod.F90 +A src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 +A src/drivers/mct/main/mrg_mod.F90 +A src/drivers/mct/main/prep_aoflux_mod.F90 +A src/drivers/mct/main/prep_atm_mod.F90 +A src/drivers/mct/main/prep_glc_mod.F90 +A src/drivers/mct/main/prep_ice_mod.F90 +A src/drivers/mct/main/prep_lnd_mod.F90 +A src/drivers/mct/main/prep_ocn_mod.F90 +A src/drivers/mct/main/prep_rof_mod.F90 +A src/drivers/mct/main/prep_wav_mod.F90 +A src/drivers/mct/main/seq_diag_mct.F90 +A src/drivers/mct/main/seq_domain_mct.F90 +A src/drivers/mct/main/seq_flux_mct.F90 +A src/drivers/mct/main/seq_frac_mct.F90 +A src/drivers/mct/main/seq_hist_mod.F90 +A src/drivers/mct/main/seq_io_mod.F90 +A src/drivers/mct/main/seq_map_mod.F90 +A src/drivers/mct/main/seq_map_type_mod.F90 +A src/drivers/mct/main/seq_rest_mod.F90 +A src/drivers/mct/main/t_driver_timers_mod.F90 +A src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 +A src/drivers/mct/main/vertical_gradient_calculator_base.F90 +A src/drivers/mct/main/vertical_gradient_calculator_factory.F90 +A src/drivers/mct/shr/CMakeLists.txt +A src/drivers/mct/shr/glc_elevclass_mod.F90 +A src/drivers/mct/shr/seq_cdata_mod.F90 +A src/drivers/mct/shr/seq_comm_mct.F90 +A src/drivers/mct/shr/seq_drydep_mod.F90 +A src/drivers/mct/shr/seq_flds_mod.F90 +A src/drivers/mct/shr/seq_infodata_mod.F90 +A src/drivers/mct/shr/seq_io_read_mod.F90 +A src/drivers/mct/shr/seq_timemgr_mod.F90 +A src/drivers/mct/shr/shr_carma_mod.F90 +A src/drivers/mct/shr/shr_expr_parser_mod.F90 +A src/drivers/mct/shr/shr_fire_emis_mod.F90 +A src/drivers/mct/shr/shr_megan_mod.F90 +A src/drivers/mct/unit_test/CMakeLists.txt +A src/drivers/mct/unit_test/avect_wrapper_test/CMakeLists.txt +A src/drivers/mct/unit_test/avect_wrapper_test/test_avect_wrapper.pf +A src/drivers/mct/unit_test/glc_elevclass_test/CMakeLists.txt +A src/drivers/mct/unit_test/glc_elevclass_test/test_glc_elevclass.pf +A src/drivers/mct/unit_test/map_glc2lnd_test/CMakeLists.txt +A src/drivers/mct/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf +A src/drivers/mct/unit_test/map_lnd2glc_test/CMakeLists.txt +A src/drivers/mct/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf +A src/drivers/mct/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt +A src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf +A src/drivers/mct/unit_test/seq_map_test/CMakeLists.txt +A src/drivers/mct/unit_test/seq_map_test/test_seq_map.pf +A src/drivers/mct/unit_test/stubs/CMakeLists.txt +A src/drivers/mct/unit_test/stubs/vertical_gradient_calculator_constant.F90 +A src/drivers/mct/unit_test/utils/CMakeLists.txt +A src/drivers/mct/unit_test/utils/avect_wrapper_mod.F90 +A src/drivers/mct/unit_test/utils/create_mapper_mod.F90 +A src/drivers/mct/unit_test/utils/mct_wrapper_mod.F90 +A src/drivers/mct/unit_test/utils/simple_map_mod.F90 +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/CMakeLists.txt +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/README +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/gradient_example.txt +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/plot_gradient +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf +A src/externals/CMake/.gitignore +A src/externals/CMake/CESM_utils.cmake +A src/externals/CMake/CIME_utils.cmake +A src/externals/CMake/ChangeLog +A src/externals/CMake/Compilers.cmake +A src/externals/CMake/FindpFUnit.cmake +A src/externals/CMake/LICENSE +A src/externals/CMake/README.md +A src/externals/CMake/Sourcelist_utils.cmake +A src/externals/CMake/TryCSizeOf.f90 +A src/externals/CMake/TryMPIIO.f90 +A src/externals/CMake/TryMPIMod.f90 +A src/externals/CMake/TryMPISERIAL.f90 +A src/externals/CMake/TryPnetcdf_inc.f90 +A src/externals/CMake/TryPnetcdf_mod.f90 +A src/externals/CMake/genf90_utils.cmake +A src/externals/CMake/mpiexec.cmake +A src/externals/CMake/pFUnit_utils.cmake +A src/externals/genf90/ChangeLog +A src/externals/genf90/genf90.pl +A src/externals/mct/.gitignore +A src/externals/mct/COPYRIGHT +A src/externals/mct/Makefile +A src/externals/mct/Makefile.conf.in +A src/externals/mct/README +A src/externals/mct/aclocal.m4 +A src/externals/mct/benchmarks/.gitignore +A src/externals/mct/benchmarks/Makefile +A src/externals/mct/benchmarks/RouterTestDis.F90 +A src/externals/mct/benchmarks/RouterTestOvr.F90 +A src/externals/mct/benchmarks/T42.8pC +A src/externals/mct/benchmarks/T42.8pR +A src/externals/mct/benchmarks/gx1.8pC +A src/externals/mct/benchmarks/gx1.8pR +A src/externals/mct/benchmarks/importBench.F90 +A src/externals/mct/config.h.in +A src/externals/mct/configure +A src/externals/mct/configure.ac +A src/externals/mct/doc/.gitignore +A src/externals/mct/doc/Makefile +A src/externals/mct/doc/README +A src/externals/mct/doc/coupler.bib +A src/externals/mct/doc/mct_APIs.tex +A src/externals/mct/doc/texsrc/.gitignore +A src/externals/mct/doc/texsrc/Makefile +A src/externals/mct/doc/texsrc/SRCS_tex.mk +A src/externals/mct/examples/Makefile +A src/externals/mct/examples/README +A src/externals/mct/examples/climate_concur1/.gitignore +A src/externals/mct/examples/climate_concur1/Makefile +A src/externals/mct/examples/climate_concur1/README +A src/externals/mct/examples/climate_concur1/coupler.F90 +A src/externals/mct/examples/climate_concur1/master.F90 +A src/externals/mct/examples/climate_concur1/model.F90 +A src/externals/mct/examples/climate_sequen1/.gitignore +A src/externals/mct/examples/climate_sequen1/Makefile +A src/externals/mct/examples/climate_sequen1/README +A src/externals/mct/examples/climate_sequen1/TS1.dat +A src/externals/mct/examples/climate_sequen1/coupler.F90 +A src/externals/mct/examples/climate_sequen1/dst.rc +A src/externals/mct/examples/climate_sequen1/dstmodel.F90 +A src/externals/mct/examples/climate_sequen1/master.F90 +A src/externals/mct/examples/climate_sequen1/mutils.F90 +A src/externals/mct/examples/climate_sequen1/src.rc +A src/externals/mct/examples/climate_sequen1/srcmodel.F90 +A src/externals/mct/examples/simple/.gitignore +A src/externals/mct/examples/simple/Makefile +A src/externals/mct/examples/simple/README +A src/externals/mct/examples/simple/script.babyblue +A src/externals/mct/examples/simple/twocmp.con.F90 +A src/externals/mct/examples/simple/twocmp.seq.F90 +A src/externals/mct/examples/simple/twocmp.seqNB.F90 +A src/externals/mct/examples/simple/twocmp.seqUnvn.F90 +A src/externals/mct/install-sh +A src/externals/mct/m4/README +A src/externals/mct/m4/acx_mpi.m4 +A src/externals/mct/m4/ax_fc_version.m4 +A src/externals/mct/m4/fortran.m4 +A src/externals/mct/mct/Makefile +A src/externals/mct/mct/README +A src/externals/mct/mct/m_Accumulator.F90 +A src/externals/mct/mct/m_AccumulatorComms.F90 +A src/externals/mct/mct/m_AttrVect.F90 +A src/externals/mct/mct/m_AttrVectComms.F90 +A src/externals/mct/mct/m_AttrVectReduce.F90 +A src/externals/mct/mct/m_ConvertMaps.F90 +A src/externals/mct/mct/m_ExchangeMaps.F90 +A src/externals/mct/mct/m_GeneralGrid.F90 +A src/externals/mct/mct/m_GeneralGridComms.F90 +A src/externals/mct/mct/m_GlobalMap.F90 +A src/externals/mct/mct/m_GlobalSegMap.F90 +A src/externals/mct/mct/m_GlobalSegMapComms.F90 +A src/externals/mct/mct/m_GlobalToLocal.F90 +A src/externals/mct/mct/m_MCTWorld.F90 +A src/externals/mct/mct/m_MatAttrVectMul.F90 +A src/externals/mct/mct/m_Merge.F90 +A src/externals/mct/mct/m_Navigator.F90 +A src/externals/mct/mct/m_Rearranger.F90 +A src/externals/mct/mct/m_Router.F90 +A src/externals/mct/mct/m_SparseMatrix.F90 +A src/externals/mct/mct/m_SparseMatrixComms.F90 +A src/externals/mct/mct/m_SparseMatrixDecomp.F90 +A src/externals/mct/mct/m_SparseMatrixPlus.F90 +A src/externals/mct/mct/m_SparseMatrixToMaps.F90 +A src/externals/mct/mct/m_SpatialIntegral.F90 +A src/externals/mct/mct/m_SpatialIntegralV.F90 +A src/externals/mct/mct/m_Transfer.F90 +A src/externals/mct/mkinstalldirs +A src/externals/mct/mpeu/Makefile +A src/externals/mct/mpeu/README +A src/externals/mct/mpeu/assertmpeu.H +A src/externals/mct/mpeu/get_zeits.c +A src/externals/mct/mpeu/m_FcComms.F90 +A src/externals/mct/mpeu/m_FileResolv.F90 +A src/externals/mct/mpeu/m_Filename.F90 +A src/externals/mct/mpeu/m_IndexBin_char.F90 +A src/externals/mct/mpeu/m_IndexBin_integer.F90 +A src/externals/mct/mpeu/m_IndexBin_logical.F90 +A src/externals/mct/mpeu/m_List.F90 +A src/externals/mct/mpeu/m_MergeSorts.F90 +A src/externals/mct/mpeu/m_Permuter.F90 +A src/externals/mct/mpeu/m_SortingTools.F90 +A src/externals/mct/mpeu/m_StrTemplate.F90 +A src/externals/mct/mpeu/m_String.F90 +A src/externals/mct/mpeu/m_StringLinkedList.F90 +A src/externals/mct/mpeu/m_TraceBack.F90 +A src/externals/mct/mpeu/m_chars.F90 +A src/externals/mct/mpeu/m_die.F90 +A src/externals/mct/mpeu/m_dropdead.F90 +A src/externals/mct/mpeu/m_flow.F90 +A src/externals/mct/mpeu/m_inpak90.F90 +A src/externals/mct/mpeu/m_ioutil.F90 +A src/externals/mct/mpeu/m_mall.F90 +A src/externals/mct/mpeu/m_mpif.F90 +A src/externals/mct/mpeu/m_mpif90.F90 +A src/externals/mct/mpeu/m_mpout.F90 +A src/externals/mct/mpeu/m_rankMerge.F90 +A src/externals/mct/mpeu/m_realkinds.F90 +A src/externals/mct/mpeu/m_stdio.F90 +A src/externals/mct/mpeu/m_zeit.F90 +A src/externals/mct/mpi-serial/.gitignore +A src/externals/mct/mpi-serial/Makefile +A src/externals/mct/mpi-serial/Makefile.conf.in +A src/externals/mct/mpi-serial/NOTES +A src/externals/mct/mpi-serial/README +A src/externals/mct/mpi-serial/aclocal.m4 +A src/externals/mct/mpi-serial/cart.c +A src/externals/mct/mpi-serial/collective.c +A src/externals/mct/mpi-serial/comm.c +A src/externals/mct/mpi-serial/config.h.in +A src/externals/mct/mpi-serial/configure +A src/externals/mct/mpi-serial/configure.in +A src/externals/mct/mpi-serial/copy.c +A src/externals/mct/mpi-serial/fort.F90 +A src/externals/mct/mpi-serial/getcount.c +A src/externals/mct/mpi-serial/group.c +A src/externals/mct/mpi-serial/handles.c +A src/externals/mct/mpi-serial/info.c +A src/externals/mct/mpi-serial/list.c +A src/externals/mct/mpi-serial/list.h +A src/externals/mct/mpi-serial/listP.h +A src/externals/mct/mpi-serial/listops.h +A src/externals/mct/mpi-serial/m4/README +A src/externals/mct/mpi-serial/m4/ax_fc_version.m4 +A src/externals/mct/mpi-serial/mpi.c +A src/externals/mct/mpi-serial/mpi.h +A src/externals/mct/mpi-serial/mpiP.h +A src/externals/mct/mpi-serial/mpif.F90 +A src/externals/mct/mpi-serial/mpif.h +A src/externals/mct/mpi-serial/op.c +A src/externals/mct/mpi-serial/pack.c +A src/externals/mct/mpi-serial/probe.c +A src/externals/mct/mpi-serial/protify.awk +A src/externals/mct/mpi-serial/recv.c +A src/externals/mct/mpi-serial/req.c +A src/externals/mct/mpi-serial/send.c +A src/externals/mct/mpi-serial/tests/.gitignore +A src/externals/mct/mpi-serial/tests/Makefile +A src/externals/mct/mpi-serial/tests/ctest.c +A src/externals/mct/mpi-serial/tests/ctest_old.c +A src/externals/mct/mpi-serial/tests/ftest.F90 +A src/externals/mct/mpi-serial/tests/ftest_internal.F90 +A src/externals/mct/mpi-serial/tests/ftest_old.F90 +A src/externals/mct/mpi-serial/time.c +A src/externals/mct/mpi-serial/type.c +A src/externals/mct/mpi-serial/type.h +A src/externals/mct/mpi-serial/type_const.c +A src/externals/mct/protex/protex +A src/externals/mct/testsystem/Makefile +A src/externals/mct/testsystem/testall/.gitignore +A src/externals/mct/testsystem/testall/Makefile +A src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 +A src/externals/mct/testsystem/testall/UNTESTED +A src/externals/mct/testsystem/testall/ccm.F90 +A src/externals/mct/testsystem/testall/convertPOPT.F90 +A src/externals/mct/testsystem/testall/convertgauss.F90 +A src/externals/mct/testsystem/testall/cpl.F90 +A src/externals/mct/testsystem/testall/job.ut-all.jaguar +A src/externals/mct/testsystem/testall/m_ACTEST.F90 +A src/externals/mct/testsystem/testall/m_AVTEST.F90 +A src/externals/mct/testsystem/testall/m_GGRIDTEST.F90 +A src/externals/mct/testsystem/testall/m_GMAPTEST.F90 +A src/externals/mct/testsystem/testall/m_GSMAPTEST.F90 +A src/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 +A src/externals/mct/testsystem/testall/m_ROUTERTEST.F90 +A src/externals/mct/testsystem/testall/m_SMATTEST.F90 +A src/externals/mct/testsystem/testall/master.F90 +A src/externals/mct/testsystem/testall/mph.F90 +A src/externals/mct/testsystem/testall/pop.F90 +A src/externals/mct/testsystem/testall/processors_map.in +A src/externals/mct/testsystem/testall/script.jag +A src/externals/mct/testsystem/testall/ut_SparseMatrix.rc +A src/externals/mct/testunit/.gitignore +A src/externals/mct/testunit/AttrVect_Test.F90 +A src/externals/mct/testunit/Makefile +A src/externals/mct/testunit/master.F90 +A src/externals/pio1/CMakeLists.txt +A src/externals/pio1/ChangeLog +A src/externals/pio1/ChangeLog_template +A src/externals/pio1/Doxyfile +A src/externals/pio1/DoxygenLayout.xml +A src/externals/pio1/SVN_EXTERNAL_DIRECTORIES +A src/externals/pio1/customdoxygen.css +A src/externals/pio1/doc/CAMexample.txt +A src/externals/pio1/doc/Decomp.txt +A src/externals/pio1/doc/DoxygenLayout.xml +A src/externals/pio1/doc/Error.txt +A src/externals/pio1/doc/Examples.txt +A src/externals/pio1/doc/GettingStarted.txt +A src/externals/pio1/doc/Installing.txt +A src/externals/pio1/doc/api.txt +A src/externals/pio1/doc/base.txt +A src/externals/pio1/doc/example/errorhandle +A src/externals/pio1/doc/example/simple-bc +A src/externals/pio1/doc/example/simple-bc-rearr +A src/externals/pio1/doc/example/simple-bc-rearr-pe1 +A src/externals/pio1/doc/example/simple-bc-rearr-pe2 +A src/externals/pio1/doc/example/simple-dof +A src/externals/pio1/doc/example/simple-dof-rearr +A src/externals/pio1/doc/faq.txt +A src/externals/pio1/doc/footer.html +A src/externals/pio1/doc/header.html +A src/externals/pio1/doc/images/baseimage.graffle +A src/externals/pio1/doc/images/block-cyclic-rearr.eps +A src/externals/pio1/doc/images/block-cyclic-rearr.graffle +A src/externals/pio1/doc/images/block-cyclic-rearr.png +A src/externals/pio1/doc/images/block-cyclic.eps +A src/externals/pio1/doc/images/block-cyclic.graffle +A src/externals/pio1/doc/images/block-cyclic.png +A src/externals/pio1/doc/images/dof-rearr.eps +A src/externals/pio1/doc/images/dof-rearr.graffle +A src/externals/pio1/doc/images/dof-rearr.png +A src/externals/pio1/doc/images/dof.eps +A src/externals/pio1/doc/images/dof.graffle +A src/externals/pio1/doc/images/dof.png +A src/externals/pio1/doc/testpio_example.txt +A src/externals/pio1/doxygen.sty +A src/externals/pio1/pio/CMakeLists.txt +A src/externals/pio1/pio/C_interface_mod.F90 +A src/externals/pio1/pio/Makefile.conf.in +A src/externals/pio1/pio/README.config +A src/externals/pio1/pio/alloc_mod.F90.in +A src/externals/pio1/pio/box_rearrange.F90.in +A src/externals/pio1/pio/calcdecomp.F90 +A src/externals/pio1/pio/calcdisplace_mod.F90 +A src/externals/pio1/pio/config.h.in +A src/externals/pio1/pio/dtypes.h +A src/externals/pio1/pio/fdepends.awk +A src/externals/pio1/pio/iompi_mod.F90.in +A src/externals/pio1/pio/ionf_mod.F90 +A src/externals/pio1/pio/nf_mod.F90 +A src/externals/pio1/pio/pio.F90 +A src/externals/pio1/pio/pio_kinds.F90 +A src/externals/pio1/pio/pio_mpi_utils.F90 +A src/externals/pio1/pio/pio_msg_callbacks.F90 +A src/externals/pio1/pio/pio_msg_getput_callbacks.F90.in +A src/externals/pio1/pio/pio_msg_mod.F90 +A src/externals/pio1/pio/pio_nf_utils.F90 +A src/externals/pio1/pio/pio_spmd_utils.F90.in +A src/externals/pio1/pio/pio_support.F90 +A src/externals/pio1/pio/pio_types.F90 +A src/externals/pio1/pio/pio_utils.F90 +A src/externals/pio1/pio/piodarray.F90.in +A src/externals/pio1/pio/piolib_mod.F90 +A src/externals/pio1/pio/pionfatt_mod.F90.in +A src/externals/pio1/pio/pionfget_mod.F90.in +A src/externals/pio1/pio/pionfput_mod.F90.in +A src/externals/pio1/pio/pionfread_mod.F90.in +A src/externals/pio1/pio/pionfwrite_mod.F90.in +A src/externals/pio1/pio/piovdc.F90 +A src/externals/pio1/pio/rearr_options.h +A src/externals/pio1/pio/rearrange.F90.in +A src/externals/pio1/pio/topology.c +A src/externals/pio1/scripts/Utils.pm +A src/externals/pio1/scripts/config.pl +A src/externals/pio1/scripts/testpio_yellowstone.pl +A src/externals/pio1/tests/testpio/CAM05.csh +A src/externals/pio1/tests/testpio/CMakeLists.txt +A src/externals/pio1/tests/testpio/MPASA30km.csh +A src/externals/pio1/tests/testpio/MPASA60km.csh +A src/externals/pio1/tests/testpio/POPB.csh +A src/externals/pio1/tests/testpio/POPC.csh +A src/externals/pio1/tests/testpio/POPD.csh +A src/externals/pio1/tests/testpio/POPDv0.csh +A src/externals/pio1/tests/testpio/POPDv1.csh +A src/externals/pio1/tests/testpio/POPDv2.csh +A src/externals/pio1/tests/testpio/POPDv3.csh +A src/externals/pio1/tests/testpio/POPDv4.csh +A src/externals/pio1/tests/testpio/POPDv5.csh +A src/externals/pio1/tests/testpio/README.testpio +A src/externals/pio1/tests/testpio/WRFB.csh +A src/externals/pio1/tests/testpio/build_defaults.xml +A src/externals/pio1/tests/testpio/check_mod.F90 +A src/externals/pio1/tests/testpio/config_bench.xml +A src/externals/pio1/tests/testpio/fdepends.awk +A src/externals/pio1/tests/testpio/gdecomp_mod.F90 +A src/externals/pio1/tests/testpio/kinds_mod.F90 +A src/externals/pio1/tests/testpio/kraken.128.csh +A src/externals/pio1/tests/testpio/kraken.1K.csh +A src/externals/pio1/tests/testpio/kraken.256.csh +A src/externals/pio1/tests/testpio/kraken.512.csh +A src/externals/pio1/tests/testpio/kraken.64.csh +A src/externals/pio1/tests/testpio/namelist_mod.F90 +A src/externals/pio1/tests/testpio/namelists/testpio_in.apb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.asb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.asb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b09 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b10 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b11 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b12 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b13 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bn01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bn02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bn03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pn01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pn02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pn03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sn01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sn02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sn03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.wr01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.wr02 +A src/externals/pio1/tests/testpio/perl5lib/ChangeLog +A src/externals/pio1/tests/testpio/perl5lib/README +A src/externals/pio1/tests/testpio/perl5lib/XML/Changes +A src/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm +A src/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm +A src/externals/pio1/tests/testpio/perl5lib/XML/README +A src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 +A src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 +A src/externals/pio1/tests/testpio/test.csh +A src/externals/pio1/tests/testpio/test_lib.F90 +A src/externals/pio1/tests/testpio/testdecomp.F90 +A src/externals/pio1/tests/testpio/testdecomp.bluefire.run +A src/externals/pio1/tests/testpio/testdecomp_in +A src/externals/pio1/tests/testpio/testpio.F90 +A src/externals/pio1/tests/testpio/testpio_bench.pl +A src/externals/pio1/tests/testpio/testpio_build.pl +A src/externals/pio1/tests/testpio/testpio_run.pl +A src/externals/pio1/tests/testpio/utils_mod.F90 +A src/externals/pio1/tests/testpio/ystest.sh +A src/externals/pio1/tests/unittests/CMakeLists.txt +A src/externals/pio1/tests/unittests/Levy_Notes +A src/externals/pio1/tests/unittests/README +A src/externals/pio1/tests/unittests/basic_tests.F90 +A src/externals/pio1/tests/unittests/driver.F90 +A src/externals/pio1/tests/unittests/global_vars.F90 +A src/externals/pio1/tests/unittests/input.nl +A src/externals/pio1/tests/unittests/nc_set_log_level2.c +A src/externals/pio1/tests/unittests/ncdf_tests.F90 +A src/externals/pio1/tests/unittests/not_netcdf.ieee +A src/externals/pio1/timing/CMakeLists.txt +A src/externals/pio1/timing/COPYING +A src/externals/pio1/timing/ChangeLog +A src/externals/pio1/timing/GPTLget_memusage.c +A src/externals/pio1/timing/GPTLprint_memusage.c +A src/externals/pio1/timing/GPTLutil.c +A src/externals/pio1/timing/Makefile +A src/externals/pio1/timing/README +A src/externals/pio1/timing/XXXdotF/perf_mod.F +A src/externals/pio1/timing/XXXdotF/perf_utils.F +A src/externals/pio1/timing/f_wrappers.c +A src/externals/pio1/timing/gptl.c +A src/externals/pio1/timing/gptl.h +A src/externals/pio1/timing/gptl.inc +A src/externals/pio1/timing/gptl_papi.c +A src/externals/pio1/timing/perf_mod.F90 +A src/externals/pio1/timing/perf_utils.F90 +A src/externals/pio1/timing/private.h +A src/externals/pio1/timing/threadutil.c +A src/externals/pio2/.gitignore +A src/externals/pio2/CMakeLists.txt +A src/externals/pio2/CTestConfig.cmake +A src/externals/pio2/CTestScript.cmake +A src/externals/pio2/README.md +A src/externals/pio2/cmake/FindGPTL.cmake +A src/externals/pio2/cmake/FindHDF5.cmake +A src/externals/pio2/cmake/FindLIBRT.cmake +A src/externals/pio2/cmake/FindLIBZ.cmake +A src/externals/pio2/cmake/FindMPE.cmake +A src/externals/pio2/cmake/FindMPISERIAL.cmake +A src/externals/pio2/cmake/FindNetCDF.cmake +A src/externals/pio2/cmake/FindPAPI.cmake +A src/externals/pio2/cmake/FindPnetCDF.cmake +A src/externals/pio2/cmake/FindSZIP.cmake +A src/externals/pio2/cmake/LibCheck.cmake +A src/externals/pio2/cmake/LibFind.cmake +A src/externals/pio2/cmake/LibMPI.cmake +A src/externals/pio2/cmake/TryHDF5_HAS_SZIP.c +A src/externals/pio2/cmake/TryNetCDF_DAP.c +A src/externals/pio2/cmake/TryNetCDF_PARALLEL.c +A src/externals/pio2/cmake/TryNetCDF_PNETCDF.c +A src/externals/pio2/cmake/mpiexec.alcf +A src/externals/pio2/cmake/mpiexec.ncsa +A src/externals/pio2/cmake/mpiexec.nersc +A src/externals/pio2/cmake/mpiexec.nwsc +A src/externals/pio2/ctest/CTestEnvironment-alcf.cmake +A src/externals/pio2/ctest/CTestEnvironment-cgd.cmake +A src/externals/pio2/ctest/CTestEnvironment-ncsa.cmake +A src/externals/pio2/ctest/CTestEnvironment-nersc.cmake +A src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +A src/externals/pio2/ctest/CTestEnvironment-unknown.cmake +A src/externals/pio2/ctest/CTestScript-Test.cmake +A src/externals/pio2/ctest/runcdash-alcf-ibm.sh +A src/externals/pio2/ctest/runcdash-cgd-nag.sh +A src/externals/pio2/ctest/runcdash-nersc-cray.sh +A src/externals/pio2/ctest/runcdash-nersc-intel.sh +A src/externals/pio2/ctest/runcdash-nwsc-gnu.sh +A src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +A src/externals/pio2/ctest/runcdash-nwsc-intel.sh +A src/externals/pio2/ctest/runcdash-nwsc-pgi.sh +A src/externals/pio2/ctest/runctest-alcf.sh +A src/externals/pio2/ctest/runctest-cgd.sh +A src/externals/pio2/ctest/runctest-ncsa.sh +A src/externals/pio2/ctest/runctest-nersc.sh +A src/externals/pio2/ctest/runctest-nwsc.sh +A src/externals/pio2/ctest/runctest-unknown.sh +A src/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake +A src/externals/pio2/doc/CMakeFiles/CMakeOutput.log +A src/externals/pio2/doc/CMakeFiles/cmake.check_cache +A src/externals/pio2/doc/CMakeLists.txt +A src/externals/pio2/doc/Doxyfile.in +A src/externals/pio2/doc/DoxygenLayout.xml +A src/externals/pio2/doc/customdoxygen.css +A src/externals/pio2/doc/doxygen.sty +A src/externals/pio2/doc/images/baseimage.graffle +A src/externals/pio2/doc/images/block-cyclic-rearr.eps +A src/externals/pio2/doc/images/block-cyclic-rearr.graffle +A src/externals/pio2/doc/images/block-cyclic-rearr.png +A src/externals/pio2/doc/images/block-cyclic.eps +A src/externals/pio2/doc/images/block-cyclic.graffle +A src/externals/pio2/doc/images/block-cyclic.png +A src/externals/pio2/doc/images/dof-rearr.eps +A src/externals/pio2/doc/images/dof-rearr.graffle +A src/externals/pio2/doc/images/dof-rearr.png +A src/externals/pio2/doc/images/dof.eps +A src/externals/pio2/doc/images/dof.graffle +A src/externals/pio2/doc/images/dof.png +A src/externals/pio2/doc/source/CAMexample.txt +A src/externals/pio2/doc/source/Decomp.txt +A src/externals/pio2/doc/source/Error.txt +A src/externals/pio2/doc/source/Examples.txt +A src/externals/pio2/doc/source/GettingStarted.txt +A src/externals/pio2/doc/source/Installing.txt +A src/externals/pio2/doc/source/api.txt +A src/externals/pio2/doc/source/base.txt +A src/externals/pio2/doc/source/example/errorhandle +A src/externals/pio2/doc/source/example/simple-bc +A src/externals/pio2/doc/source/example/simple-bc-rearr +A src/externals/pio2/doc/source/example/simple-bc-rearr-pe1 +A src/externals/pio2/doc/source/example/simple-bc-rearr-pe2 +A src/externals/pio2/doc/source/example/simple-dof +A src/externals/pio2/doc/source/example/simple-dof-rearr +A src/externals/pio2/doc/source/faq.txt +A src/externals/pio2/doc/source/mach_walkthrough.txt +A src/externals/pio2/doc/source/testpio_example.txt +A src/externals/pio2/examples/CMakeLists.txt +A src/externals/pio2/examples/basic/CAM05.csh +A src/externals/pio2/examples/basic/CMakeLists.txt +A src/externals/pio2/examples/basic/MPASA30km.csh +A src/externals/pio2/examples/basic/MPASA60km.csh +A src/externals/pio2/examples/basic/POPB.csh +A src/externals/pio2/examples/basic/POPC.csh +A src/externals/pio2/examples/basic/POPD.csh +A src/externals/pio2/examples/basic/POPDv0.csh +A src/externals/pio2/examples/basic/POPDv1.csh +A src/externals/pio2/examples/basic/POPDv2.csh +A src/externals/pio2/examples/basic/POPDv3.csh +A src/externals/pio2/examples/basic/POPDv4.csh +A src/externals/pio2/examples/basic/POPDv5.csh +A src/externals/pio2/examples/basic/README.testpio +A src/externals/pio2/examples/basic/WRFB.csh +A src/externals/pio2/examples/basic/alloc_mod.F90.in +A src/externals/pio2/examples/basic/build_defaults.xml +A src/externals/pio2/examples/basic/check_mod.F90 +A src/externals/pio2/examples/basic/config_bench.xml +A src/externals/pio2/examples/basic/fdepends.awk +A src/externals/pio2/examples/basic/gdecomp_mod.F90 +A src/externals/pio2/examples/basic/kinds_mod.F90 +A src/externals/pio2/examples/basic/kraken.128.csh +A src/externals/pio2/examples/basic/kraken.1K.csh +A src/externals/pio2/examples/basic/kraken.256.csh +A src/externals/pio2/examples/basic/kraken.512.csh +A src/externals/pio2/examples/basic/kraken.64.csh +A src/externals/pio2/examples/basic/namelist_mod.F90 +A src/externals/pio2/examples/basic/namelists/testpio_in.apb05 +A src/externals/pio2/examples/basic/namelists/testpio_in.asb01 +A src/externals/pio2/examples/basic/namelists/testpio_in.asb04 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b01 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b02 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b03 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b04 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b05 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b06 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b07 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b08 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4n01 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4n02 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4n03 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb01 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb02 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb03 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb04 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb05 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb06 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb07 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb08 +A src/externals/pio2/examples/basic/namelists/testpio_in.pn01 +A src/externals/pio2/examples/basic/namelists/testpio_in.pn02 +A src/externals/pio2/examples/basic/namelists/testpio_in.pn03 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps01 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps02 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps03 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps04 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps05 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps06 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps07 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps08 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb01 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb02 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb03 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb04 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb05 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb06 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb07 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb08 +A src/externals/pio2/examples/basic/namelists/testpio_in.sn01 +A src/externals/pio2/examples/basic/namelists/testpio_in.sn02 +A src/externals/pio2/examples/basic/namelists/testpio_in.sn03 +A src/externals/pio2/examples/basic/namelists/testpio_in.wr01 +A src/externals/pio2/examples/basic/namelists/testpio_in.wr02 +A src/externals/pio2/examples/basic/perl5lib/ChangeLog +A src/externals/pio2/examples/basic/perl5lib/README +A src/externals/pio2/examples/basic/perl5lib/XML/Changes +A src/externals/pio2/examples/basic/perl5lib/XML/Lite.pm +A src/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm +A src/externals/pio2/examples/basic/perl5lib/XML/README +A src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 +A src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 +A src/externals/pio2/examples/basic/test.csh +A src/externals/pio2/examples/basic/testdecomp.F90 +A src/externals/pio2/examples/basic/testdecomp.bluefire.run +A src/externals/pio2/examples/basic/testdecomp_in +A src/externals/pio2/examples/basic/testpio.F90 +A src/externals/pio2/examples/basic/testpio_bench.pl +A src/externals/pio2/examples/basic/testpio_build.pl +A src/externals/pio2/examples/basic/testpio_run.pl +A src/externals/pio2/examples/basic/utils_mod.F90 +A src/externals/pio2/examples/basic/wstest.c +A src/externals/pio2/examples/basic/ystest.sh +A src/externals/pio2/examples/c/CMakeLists.txt +A src/externals/pio2/examples/c/example1.c +A src/externals/pio2/examples/c/example2.c +A src/externals/pio2/examples/c/examplePio.c +A src/externals/pio2/examples/cxx/CMakeLists.txt +A src/externals/pio2/examples/cxx/examplePio.cxx +A src/externals/pio2/examples/f03/CMakeLists.txt +A src/externals/pio2/examples/f03/examplePio.f90 +A src/externals/pio2/scripts/prune_decomps.pl +A src/externals/pio2/src/CMakeLists.txt +A src/externals/pio2/src/clib/CMakeLists.txt +A src/externals/pio2/src/clib/bget.c +A src/externals/pio2/src/clib/bget.h +A src/externals/pio2/src/clib/config.h.in +A src/externals/pio2/src/clib/dtypes.h +A src/externals/pio2/src/clib/ncparser.pl +A src/externals/pio2/src/clib/ncputgetparser.pl +A src/externals/pio2/src/clib/pio.h +A src/externals/pio2/src/clib/pio_c_get_template.c +A src/externals/pio2/src/clib/pio_c_put_template.c +A src/externals/pio2/src/clib/pio_c_template.c +A src/externals/pio2/src/clib/pio_darray.c +A src/externals/pio2/src/clib/pio_file.c +A src/externals/pio2/src/clib/pio_get_nc.c +A src/externals/pio2/src/clib/pio_internal.h +A src/externals/pio2/src/clib/pio_lists.c +A src/externals/pio2/src/clib/pio_nc.c +A src/externals/pio2/src/clib/pio_nc4.c +A src/externals/pio2/src/clib/pio_put_nc.c +A src/externals/pio2/src/clib/pio_rearrange.c +A src/externals/pio2/src/clib/pio_spmd.c +A src/externals/pio2/src/clib/pioc.c +A src/externals/pio2/src/clib/pioc_sc.c +A src/externals/pio2/src/clib/pioc_support.c +A src/externals/pio2/src/clib/topology.c +A src/externals/pio2/src/flib/CMakeLists.txt +A src/externals/pio2/src/flib/pio.F90 +A src/externals/pio2/src/flib/pio_kinds.F90 +A src/externals/pio2/src/flib/pio_nf.F90 +A src/externals/pio2/src/flib/pio_support.F90 +A src/externals/pio2/src/flib/pio_types.F90 +A src/externals/pio2/src/flib/piodarray.F90.in +A src/externals/pio2/src/flib/piolib_mod.F90 +A src/externals/pio2/src/flib/pionfatt_mod.F90.in +A src/externals/pio2/src/flib/pionfget_mod.F90.in +A src/externals/pio2/src/flib/pionfput_mod.F90.in +A src/externals/pio2/src/gptl/CMakeLists.txt +A src/externals/pio2/src/gptl/COPYING +A src/externals/pio2/src/gptl/ChangeLog +A src/externals/pio2/src/gptl/GPTLget_memusage.c +A src/externals/pio2/src/gptl/GPTLprint_memusage.c +A src/externals/pio2/src/gptl/GPTLutil.c +A src/externals/pio2/src/gptl/README +A src/externals/pio2/src/gptl/f_wrappers.c +A src/externals/pio2/src/gptl/gptl.c +A src/externals/pio2/src/gptl/gptl.h +A src/externals/pio2/src/gptl/gptl.inc +A src/externals/pio2/src/gptl/gptl_papi.c +A src/externals/pio2/src/gptl/perf_mod.F90 +A src/externals/pio2/src/gptl/perf_utils.F90 +A src/externals/pio2/src/gptl/private.h +A src/externals/pio2/src/gptl/threadutil.c +A src/externals/pio2/tests/CMakeLists.txt +A src/externals/pio2/tests/general/CMakeLists.txt +A src/externals/pio2/tests/general/ncdf_fail.F90.in +A src/externals/pio2/tests/general/ncdf_get_put.F90.in +A src/externals/pio2/tests/general/ncdf_simple_tests.F90.in +A src/externals/pio2/tests/general/pio_decomp_fillval.F90.in +A src/externals/pio2/tests/general/pio_decomp_fillval2.F90 +A src/externals/pio2/tests/general/pio_decomp_tests.F90.in +A src/externals/pio2/tests/general/pio_fail.F90.in +A src/externals/pio2/tests/general/pio_file_fail.F90.in +A src/externals/pio2/tests/general/pio_file_simple_tests.F90.in +A src/externals/pio2/tests/general/pio_init_finalize.F90.in +A src/externals/pio2/tests/general/test_memleak.c +A src/externals/pio2/tests/general/util/pio_tf_f90gen.pl +A src/externals/pio2/tests/general/util/pio_tutil.F90 +A src/externals/pio2/tests/performance/CMakeLists.txt +A src/externals/pio2/tests/performance/gensimple.pl +A src/externals/pio2/tests/performance/kt.PIO1.perfmakefile +A src/externals/pio2/tests/performance/pioperformance.F90 +A src/externals/pio2/tests/unit/CMakeLists.txt +A src/externals/pio2/tests/unit/Levy_Notes +A src/externals/pio2/tests/unit/README +A src/externals/pio2/tests/unit/basic_tests.F90 +A src/externals/pio2/tests/unit/driver.F90 +A src/externals/pio2/tests/unit/global_vars.F90 +A src/externals/pio2/tests/unit/input.nl +A src/externals/pio2/tests/unit/nc_set_log_level2.c +A src/externals/pio2/tests/unit/ncdf_tests.F90 +A src/externals/pio2/tests/unit/not_netcdf.ieee +A src/externals/pio2/tests/unit/test_names.c +A src/externals/pio2/tests/unit/test_nc4.c +A src/share/README +A src/share/RandNum/include/dSFMT-common.h +A src/share/RandNum/include/dSFMT-params.h +A src/share/RandNum/include/dSFMT-params19937.h +A src/share/RandNum/include/dSFMT.h +A src/share/RandNum/src/dsfmt_f03/dSFMT.c +A src/share/RandNum/src/dsfmt_f03/dSFMT_interface.F90 +A src/share/RandNum/src/dsfmt_f03/dSFMT_utils.c +A src/share/RandNum/src/kissvec/kissvec.c +A src/share/RandNum/src/kissvec/kissvec_mod.F90 +A src/share/RandNum/src/mt19937/mersennetwister_mod.F90 +A src/share/RandNum/src/shr_RandNum_mod.F90 +A src/share/RandNum/test/bench/Makefile +A src/share/RandNum/test/bench/test_shr_RandNum.F90 +A src/share/esmf_wrf_timemgr/CMakeLists.txt +A src/share/esmf_wrf_timemgr/ESMF.F90 +A src/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_BaseMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_ClockMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_FractionMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_Macros.inc +A src/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_Stubs.F90 +A src/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc +A src/share/esmf_wrf_timemgr/ESMF_TimeMod.F90 +A src/share/esmf_wrf_timemgr/Makefile +A src/share/esmf_wrf_timemgr/MeatMod.F90 +A src/share/esmf_wrf_timemgr/README +A src/share/esmf_wrf_timemgr/unittests/Makefile +A src/share/esmf_wrf_timemgr/unittests/go.csh +A src/share/esmf_wrf_timemgr/unittests/test.F90 +A src/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90 +A src/share/esmf_wrf_timemgr/wrf_error_fatal.F90 +A src/share/esmf_wrf_timemgr/wrf_message.F90 +A src/share/include/dynamic_vector_procdef.inc +A src/share/include/dynamic_vector_typedef.inc +A src/share/include/shr_assert.h +A src/share/test/old_unit_testers/Makefile +A src/share/test/old_unit_testers/Mkdepends +A src/share/test/old_unit_testers/Mksrcfiles +A src/share/test/old_unit_testers/bundle_expected.F90 +A src/share/test/old_unit_testers/config.h +A src/share/test/old_unit_testers/make.Macros +A src/share/test/old_unit_testers/namelist +A src/share/test/old_unit_testers/nl/atm.stdin +A src/share/test/old_unit_testers/nl/cpl.stdin +A src/share/test/old_unit_testers/nl/ice.stdin +A src/share/test/old_unit_testers/nl/lnd.stdin +A src/share/test/old_unit_testers/nl/ocn.stdin +A src/share/test/old_unit_testers/run_dshr_bundle_test +A src/share/test/old_unit_testers/run_file_test +A src/share/test/old_unit_testers/test_mod.F90 +A src/share/test/old_unit_testers/test_shr_file.F90 +A src/share/test/old_unit_testers/test_shr_log.F90 +A src/share/test/old_unit_testers/test_shr_mpi.F90 +A src/share/test/old_unit_testers/test_shr_orb.F90 +A src/share/test/old_unit_testers/test_shr_scam.F90 +A src/share/test/old_unit_testers/test_shr_streams.F90 +A src/share/test/old_unit_testers/test_shr_sys.F90 +A src/share/test/old_unit_testers/test_shr_tInterp.F90 +A src/share/test/unit/CMakeLists.txt +A src/share/test/unit/dynamic_vector/CMakeLists.txt +A src/share/test/unit/dynamic_vector/character16_vector_tests.pf.in +A src/share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc +A src/share/test/unit/dynamic_vector/dynamic_vector_character16.F90 +A src/share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 +A src/share/test/unit/dynamic_vector/dynamic_vector_integer.F90 +A src/share/test/unit/dynamic_vector/dynamic_vector_r8.F90 +A src/share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in +A src/share/test/unit/dynamic_vector/integer_vector_tests.pf.in +A src/share/test/unit/dynamic_vector/ptr_wrapper.F90 +A src/share/test/unit/dynamic_vector/r8_vector_tests.pf.in +A src/share/test/unit/mock/CMakeLists.txt +A src/share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 +A src/share/test/unit/shr_assert_test/CMakeLists.txt +A src/share/test/unit/shr_assert_test/test_assert.pf +A src/share/test/unit/shr_assert_test/test_assert_array.pf +A src/share/test/unit/shr_assert_test/test_macro.pf +A src/share/test/unit/shr_assert_test/test_ndebug.pf +A src/share/test/unit/shr_infnan_test/CMakeLists.txt +A src/share/test/unit/shr_infnan_test/test_infnan.F90 +A src/share/test/unit/shr_log_test/CMakeLists.txt +A src/share/test/unit/shr_log_test/test_error_printers.pf +A src/share/test/unit/shr_precip_test/CMakeLists.txt +A src/share/test/unit/shr_precip_test/test_shr_precip.pf +A src/share/test/unit/shr_spfn_test/CMakeLists.txt +A src/share/test/unit/shr_spfn_test/test_erf_r4.pf +A src/share/test/unit/shr_spfn_test/test_erf_r8.pf +A src/share/test/unit/shr_spfn_test/test_gamma_factorial.pf +A src/share/test/unit/shr_spfn_test/test_igamma.pf +A src/share/test/unit/shr_strconvert_test/CMakeLists.txt +A src/share/test/unit/shr_strconvert_test/test_toString.pf +A src/share/test/unit/shr_string_test/CMakeLists.txt +A src/share/test/unit/shr_string_test/test_shr_string.pf +A src/share/test/unit/shr_vmath_test/CMakeLists.txt +A src/share/test/unit/shr_vmath_test/test_vmath.F90 +A src/share/test/unit/shr_wv_sat_test/CMakeLists.txt +A src/share/test/unit/shr_wv_sat_test/test_wv_sat.pf +A src/share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf +A src/share/timing/CMakeLists.txt +A src/share/timing/COPYING +A src/share/timing/ChangeLog +A src/share/timing/GPTLget_memusage.c +A src/share/timing/GPTLprint_memusage.c +A src/share/timing/GPTLutil.c +A src/share/timing/Makefile +A src/share/timing/README +A src/share/timing/f_wrappers.c +A src/share/timing/gptl.c +A src/share/timing/gptl.h +A src/share/timing/gptl.inc +A src/share/timing/gptl_papi.c +A src/share/timing/perf_mod.F90 +A src/share/timing/perf_utils.F90 +A src/share/timing/private.h +A src/share/unit_test_stubs/README +A src/share/unit_test_stubs/pio/CMakeLists.txt +A src/share/unit_test_stubs/pio/README +A src/share/unit_test_stubs/pio/pio.F90.in +A src/share/util/CMakeLists.txt +A src/share/util/mct_mod.F90 +A src/share/util/shr_assert_mod.F90.in +A src/share/util/shr_cal_mod.F90 +A src/share/util/shr_const_mod.F90 +A src/share/util/shr_dmodel_mod.F90 +A src/share/util/shr_file_mod.F90 +A src/share/util/shr_flux_mod.F90 +A src/share/util/shr_frz_mod.F90.in +A src/share/util/shr_infnan_mod.F90.in +A src/share/util/shr_kind_mod.F90 +A src/share/util/shr_log_mod.F90 +A src/share/util/shr_map_mod.F90 +A src/share/util/shr_mct_mod.F90 +A src/share/util/shr_mem_mod.F90 +A src/share/util/shr_mpi_mod.F90 +A src/share/util/shr_msg_mod.F90 +A src/share/util/shr_ncread_mod.F90 +A src/share/util/shr_nl_mod.F90 +A src/share/util/shr_orb_mod.F90 +A src/share/util/shr_pcdf_mod.F90 +A src/share/util/shr_pio_mod.F90 +A src/share/util/shr_precip_mod.F90 +A src/share/util/shr_reprosum_mod.F90 +A src/share/util/shr_reprosumx86.c +A src/share/util/shr_scam_mod.F90 +A src/share/util/shr_spfn_mod.F90 +A src/share/util/shr_strconvert_mod.F90 +A src/share/util/shr_strdata_mod.F90 +A src/share/util/shr_stream_mod.F90 +A src/share/util/shr_string_mod.F90 +A src/share/util/shr_sys_mod.F90 +A src/share/util/shr_tInterp_mod.F90 +A src/share/util/shr_timer_mod.F90 +A src/share/util/shr_vmath_mod.F90 +A src/share/util/shr_wv_sat_mod.F90 +A src/share/util/water_isotopes.F90 +A src/share/util/water_types.F90 +D tests/unit/test_names.c +M tools/configure +M tools/cprnc/CMakeLists.txt +M tools/cprnc/Makefile +M tools/unit_testing/run_tests.py +D utils/data_assimilation/da_no_data_mod.sh +M utils/perl5lib/compilers_translation_tool.pl +D utils/python/CIME/BuildTools/__init__.py +D utils/python/CIME/BuildTools/cmakemacroswriter.py +D utils/python/CIME/BuildTools/configure.py +D utils/python/CIME/BuildTools/macroconditiontree.py +D utils/python/CIME/BuildTools/macrowriterbase.py +D utils/python/CIME/BuildTools/makemacroswriter.py +D utils/python/CIME/BuildTools/possiblevalues.py +D utils/python/CIME/BuildTools/valuesetting.py +D utils/python/CIME/SystemTests/README +D utils/python/CIME/SystemTests/__init__.py +D utils/python/CIME/SystemTests/dae.py +D utils/python/CIME/SystemTests/eri.py +D utils/python/CIME/SystemTests/erio.py +D utils/python/CIME/SystemTests/erp.py +D utils/python/CIME/SystemTests/err.py +D utils/python/CIME/SystemTests/ers.py +D utils/python/CIME/SystemTests/ert.py +D utils/python/CIME/SystemTests/homme.py +D utils/python/CIME/SystemTests/icp.py +D utils/python/CIME/SystemTests/lii.py +D utils/python/CIME/SystemTests/nck.py +D utils/python/CIME/SystemTests/ncr.py +D utils/python/CIME/SystemTests/pea.py +D utils/python/CIME/SystemTests/pem.py +D utils/python/CIME/SystemTests/pet.py +D utils/python/CIME/SystemTests/pfs.py +D utils/python/CIME/SystemTests/pre.py +D utils/python/CIME/SystemTests/rep.py +D utils/python/CIME/SystemTests/seq.py +D utils/python/CIME/SystemTests/sms.py +D utils/python/CIME/SystemTests/ssp.py +D utils/python/CIME/SystemTests/system_tests_common.py +D utils/python/CIME/SystemTests/system_tests_compare_two.py +D utils/python/CIME/SystemTests/test_utils/__init__.py +D utils/python/CIME/SystemTests/test_utils/user_nl_utils.py +D utils/python/CIME/XML/__init__.py +D utils/python/CIME/XML/archive.py +D utils/python/CIME/XML/batch.py +D utils/python/CIME/XML/compilerblock.py +D utils/python/CIME/XML/compilers.py +D utils/python/CIME/XML/component.py +D utils/python/CIME/XML/compsets.py +D utils/python/CIME/XML/entry_id.py +D utils/python/CIME/XML/env_archive.py +D utils/python/CIME/XML/env_base.py +D utils/python/CIME/XML/env_batch.py +D utils/python/CIME/XML/env_build.py +D utils/python/CIME/XML/env_case.py +D utils/python/CIME/XML/env_mach_pes.py +D utils/python/CIME/XML/env_mach_specific.py +D utils/python/CIME/XML/env_run.py +D utils/python/CIME/XML/env_test.py +D utils/python/CIME/XML/files.py +D utils/python/CIME/XML/generic_xml.py +D utils/python/CIME/XML/grids.py +D utils/python/CIME/XML/headers.py +D utils/python/CIME/XML/lt_archive.py +D utils/python/CIME/XML/machines.py +D utils/python/CIME/XML/namelist_definition.py +D utils/python/CIME/XML/pes.py +D utils/python/CIME/XML/pio.py +D utils/python/CIME/XML/standard_module_setup.py +D utils/python/CIME/XML/testlist.py +D utils/python/CIME/XML/tests.py +D utils/python/CIME/XML/testspec.py +D utils/python/CIME/__init__.py +D utils/python/CIME/bless_test_results.py +D utils/python/CIME/build.py +D utils/python/CIME/buildlib.py +D utils/python/CIME/buildnml.py +D utils/python/CIME/case.py +D utils/python/CIME/case_cmpgen_namelists.py +D utils/python/CIME/case_lt_archive.py +D utils/python/CIME/case_run.py +D utils/python/CIME/case_setup.py +D utils/python/CIME/case_st_archive.py +D utils/python/CIME/case_submit.py +D utils/python/CIME/case_test.py +D utils/python/CIME/check_input_data.py +D utils/python/CIME/check_lockedfiles.py +D utils/python/CIME/code_checker.py +D utils/python/CIME/compare_namelists.py +D utils/python/CIME/compare_test_results.py +D utils/python/CIME/get_timing.py +D utils/python/CIME/hist_utils.py +D utils/python/CIME/namelist.py +D utils/python/CIME/nmlgen.py +D utils/python/CIME/preview_namelists.py +D utils/python/CIME/provenance.py +D utils/python/CIME/simple_compare.py +D utils/python/CIME/test_scheduler.py +D utils/python/CIME/test_status.py +D utils/python/CIME/test_utils.py +D utils/python/CIME/tests/README +D utils/python/CIME/tests/SystemTests/__init__.py +D utils/python/CIME/tests/SystemTests/test_system_tests_compare_two.py +D utils/python/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py +D utils/python/CIME/tests/SystemTests/test_utils/__init__.py +D utils/python/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py +D utils/python/CIME/tests/__init__.py +D utils/python/CIME/tests/case_fake.py +D utils/python/CIME/tests/test_case_fake.py +D utils/python/CIME/user_mod_support.py +D utils/python/CIME/utils.py +D utils/python/CIME/wait_for_tests.py +D utils/python/cs.status.template +D utils/python/cs.submit.template +D utils/python/jenkins_generic_job.py +D utils/python/tests/CMakeLists.txt +D utils/python/tests/CTestConfig.cmake +D utils/python/tests/cpl.hi1.nc.test +D utils/python/tests/cpl.hi2.nc.test +D utils/python/tests/cpl.log.failmemleak.gz +D utils/python/tests/cpl.log.passmemleak.gz +D utils/python/tests/list_tests +D utils/python/tests/scripts_regression_tests.py +D utils/python/tests/user_mods_test1/include_user_mods +D utils/python/tests/user_mods_test1/user_nl_comp +D utils/python/tests/user_mods_test1/user_nl_cpl +D utils/python/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 +D utils/python/tests/user_mods_test2/user_nl_cpl +D utils/python/update_acme_tests.py + +====================================================================== + +====================================================================== + Originator: Chris Fischer Date: 2-23-2017 Tag: cime5.2.0-alpha.26 diff --git a/README.unit_testing b/README.unit_testing index 7b2a3d1983e..df92424f99d 100644 --- a/README.unit_testing +++ b/README.unit_testing @@ -13,4 +13,4 @@ # https://sourceforge.net/projects/pfunit/ # -tools/unit_testing/run_tests.py --test-spec-dir=. --compiler=intel --mpilib=mpich2 --use-openmp --mpirun-command=mpirun.lsf --build-dir `mktemp -d --tmpdir=. unit_tests.XXXXXXXX` +tools/unit_testing/run_tests.py --build-dir `mktemp -d ./unit_tests.XXXXXXXX` diff --git a/cime_config/acme/allactive/config_compsets.xml b/config/acme/allactive/config_compsets.xml similarity index 100% rename from cime_config/acme/allactive/config_compsets.xml rename to config/acme/allactive/config_compsets.xml diff --git a/cime_config/acme/allactive/config_compsets.xml.cime2 b/config/acme/allactive/config_compsets.xml.cime2 similarity index 100% rename from cime_config/acme/allactive/config_compsets.xml.cime2 rename to config/acme/allactive/config_compsets.xml.cime2 diff --git a/cime_config/acme/allactive/config_pes.xml b/config/acme/allactive/config_pes.xml similarity index 97% rename from cime_config/acme/allactive/config_pes.xml rename to config/acme/allactive/config_pes.xml index 820aa89ae0a..d64ba8c9ac6 100644 --- a/cime_config/acme/allactive/config_pes.xml +++ b/config/acme/allactive/config_pes.xml @@ -1,4 +1,6 @@ + + @@ -185,6 +187,43 @@ + + + + none + + -16 + -9 + -9 + -7 + -1 + -1 + -1 + -16 + + + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + + + 0 + 0 + 0 + -9 + -16 + 0 + 0 + 0 + + + + @@ -1369,6 +1408,43 @@ + + + + none + + -16 + -9 + -9 + -7 + -1 + -1 + -1 + -16 + + + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + + + 0 + 0 + 0 + -9 + -16 + 0 + 0 + 0 + + + + diff --git a/cime_config/acme/allactive/config_pes.xml.cime2 b/config/acme/allactive/config_pes.xml.cime2 similarity index 100% rename from cime_config/acme/allactive/config_pes.xml.cime2 rename to config/acme/allactive/config_pes.xml.cime2 diff --git a/cime_config/acme/allactive/config_pesall.xml b/config/acme/allactive/config_pesall.xml similarity index 100% rename from cime_config/acme/allactive/config_pesall.xml rename to config/acme/allactive/config_pesall.xml diff --git a/cime_config/acme/allactive/testlist_allactive.xml b/config/acme/allactive/testlist_allactive.xml similarity index 100% rename from cime_config/acme/allactive/testlist_allactive.xml rename to config/acme/allactive/testlist_allactive.xml diff --git a/cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands b/config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands similarity index 100% rename from cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands rename to config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands diff --git a/cime_config/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands b/config/acme/allactive/testmods_dirs/force_netcdf_pio/shell_commands similarity index 100% rename from cime_config/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands rename to config/acme/allactive/testmods_dirs/force_netcdf_pio/shell_commands diff --git a/cime_config/acme/allactive/usermods_dirs/b1850/shell_commands b/config/acme/allactive/usermods_dirs/b1850/shell_commands similarity index 100% rename from cime_config/acme/allactive/usermods_dirs/b1850/shell_commands rename to config/acme/allactive/usermods_dirs/b1850/shell_commands diff --git a/cime_config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods b/config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods similarity index 100% rename from cime_config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods rename to config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods diff --git a/cime_config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism b/config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism similarity index 100% rename from cime_config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism rename to config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism diff --git a/cime_config/acme/config_archive.xml b/config/acme/config_archive.xml similarity index 65% rename from cime_config/acme/config_archive.xml rename to config/acme/config_archive.xml index a4843a2b572..1f4a64cc94f 100644 --- a/cime_config/acme/config_archive.xml +++ b/config/acme/config_archive.xml @@ -11,6 +11,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.atm + $CASE.datm$NINST_STRING.r.$DATENAME.nc,$CASE.datm$NINST_STRING.rs1.$DATENAME.bin + + + \.[ri]\..* \.rh.* @@ -22,6 +31,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.lnd + $CASE.dlnd$NINST_STRING.r.$DATENAME.nc,$CASE.dlnd$NINST_STRING.rs1.$DATENAME.bin + + + \.r.* \.h.*.nc$ @@ -42,6 +60,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.rof + $CASE.drof$NINST_STRING.r.$DATENAME.nc,$CASE.drof$NINST_STRING.rs1.$DATENAME.bin + + + \.[ri].* \.h.*.nc$ @@ -61,6 +88,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.ice + $CASE.dice$NINST_STRING.r.$DATENAME.nc,$CASE.dice$NINST_STRING.rs1.$DATENAME.bin + + + \.r.* \.h.*.nc$|\.d[dovt]\. @@ -84,6 +120,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.ocn + $CASE.docn$NINST_STRING.r.$DATENAME.nc,$CASE.docn$NINST_STRING.rs1.$DATENAME.bin + + + \.[ri]\..* \.h\..*\.nc$ @@ -114,6 +159,25 @@ + + \.r.* + unset + + rpointer$NINST_STRING.wav + $CASE.dwav$NINST_STRING.r.$DATENAME.nc,$CASE.dwav$NINST_STRING.rs1.$DATENAME.bin + + + + + \.r\..* + \.h.*.nc$ + unset + + rpointer.cpl + $CASE.cpl.r.$DATENAME.nc + + + inflate_restart.* \.True_State.* diff --git a/cime_config/acme/config_files.xml b/config/acme/config_files.xml similarity index 61% rename from cime_config/acme/config_files.xml rename to config/acme/config_files.xml index dcd98320309..97867caa304 100644 --- a/cime_config/acme/config_files.xml +++ b/config/acme/config_files.xml @@ -18,7 +18,7 @@ char - $CIMEROOT/cime_config/config_headers.xml + $CIMEROOT/config/config_headers.xml case_der env_case.xml contains both header and group information for all the case env_*.xml files @@ -26,16 +26,16 @@ char - $CIMEROOT/cime_config/$MODEL/machines/config_batch.xml + $CIMEROOT/config/$MODEL/machines/config_batch.xml case_last env_case.xml file containing batch system details for target system (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_batch.xsd + $CIMEROOT/config/xml_schemas/config_batch.xsd char - $CIMEROOT/cime_config/$MODEL/config_grids.xml + $CIMEROOT/config/$MODEL/config_grids.xml case_last env_case.xml file containing specification of all supported model grids, domains and mapping files (for documentation only - DO NOT EDIT) @@ -43,16 +43,16 @@ char - $CIMEROOT/cime_config/$MODEL/machines/config_compilers.xml + $CIMEROOT/config/$MODEL/machines/config_compilers.xml case_last env_case.xml file containing compiler specifications for target model primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_compilers_v2.xsd + $CIMEROOT/config/xml_schemas/config_compilers_v2.xsd char - $CIMEROOT/cime_config/$MODEL/machines/config_machines.xml + $CIMEROOT/config/$MODEL/machines/config_machines.xml case_last env_case.xml file containing machine specifications for target model primary component (for documentation only - DO NOT EDIT) @@ -60,7 +60,7 @@ char - $CIMEROOT/cime_config/$MODEL/machines/config_pio.xml + $CIMEROOT/config/$MODEL/machines/config_pio.xml case_last env_case.xml file containing specification of pio settings for target model possible machine, compiler, mpilib, compset and/or grid attributes (for documentation only - DO NOT EDIT) @@ -68,7 +68,7 @@ char - $CIMEROOT/cime_config/config_tests.xml + $CIMEROOT/config/config_tests.xml test env_test.xml file containing system test descriptions @@ -83,8 +83,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/config_compsets.xml - $CIMEROOT/driver_cpl/cime_config/config_compsets.xml + $CIMEROOT/config/$MODEL/allactive/config_compsets.xml + $CIMEROOT/src/drivers/mct/cime_config/config_compsets.xml $SRCROOT/components/cam/cime_config/config_compsets.xml $SRCROOT/components/cism/cime_config/config_compsets.xml $SRCROOT/components/clm/cime_config/config_compsets.xml @@ -97,62 +97,58 @@ case_last env_case.xml file containing specification of all compsets for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_compsets.xsd + $CIMEROOT/config/xml_schemas/config_compsets.xsd char unset - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/driver_cpl/cime_config/config_pes.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/src/drivers/mct/cime_config/config_pes.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml case_last env_case.xml file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_pes.xsd + $CIMEROOT/config/xml_schemas/config_pes.xsd - char - $CIMEROOT/cime_config/acme/config_archive.xml - $CIMEROOT/driver_cpl/cime_config/config_archive.xml + $CIMEROOT/config/acme/config_archive.xml + $CIMEROOT/src/drivers/mct/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/drof/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/datm/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dice/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dlnd/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/docn/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dwav/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/drof/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/datm/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dice/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dlnd/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/docn/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dwav/cime_config/config_archive.xml $SRCROOT/components/cam/cime_config/config_archive.xml $SRCROOT/components/cism/cime_config/config_archive.xml $SRCROOT/components/clm/cime_config/config_archive.xml $SRCROOT/components/cice/cime_config/config_archive.xml $SRCROOT/components/pop/cime_config/config_archive.xml - $SRCROOT/components/mpas-o/cime_config/config_archive.xml - $SRCROOT/components/mpasli/cime_config/config_archive.xml - $SRCROOT/components/mpas-cice/cime_config/config_archive.xml case_last env_case.xml file containing specification of archive files for each component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_archive.xsd + $CIMEROOT/config/xml_schemas/config_archive.xsd char - $CIMEROOT/utils/python/CIME/SystemTests + $CIMEROOT/scripts/lib/CIME/SystemTests $SRCROOT/components/clm/cime_config/SystemTests $SRCROOT/components/cam/cime_config/SystemTests $SRCROOT/components/pop/cime_config/SystemTests @@ -168,8 +164,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/testlist_allactive.xml - $CIMEROOT/driver_cpl/cime_config/testdefs/testlist_drv.xml + $CIMEROOT/config/$MODEL/allactive/testlist_allactive.xml + $CIMEROOT/src/drivers/mct/cime_config/testdefs/testlist_drv.xml $SRCROOT/components/cam/cime_config/testdefs/testlist_cam.xml $SRCROOT/components/cism/cime_config/testdefs/testlist_cism.xml $SRCROOT/components/clm/cime_config/testdefs/testlist_clm.xml @@ -187,15 +183,15 @@ char unset - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs case_last env_case.xml @@ -206,8 +202,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/usermods_dirs - $CIMEROOT/driver_cpl/cime_config/usermods_dirs + $CIMEROOT/config/$MODEL/allactive/usermods_dirs + $CIMEROOT/src/drivers/mct/cime_config/usermods_dirs $SRCROOT/components/cam/cime_config/usermods_dirs $SRCROOT/components/cism/cime_config/usermods_dirs $SRCROOT/components/clm/cime_config/usermods_dirs @@ -225,16 +221,16 @@ char unset - $CIMEROOT/driver_cpl/cime_config/namelist_definition_modelio.xml - $CIMEROOT/driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml - $CIMEROOT/driver_cpl/cime_config/namelist_definition_drv.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_modelio.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv.xml - $CIMEROOT/components/data_comps/drof/cime_config/namelist_definition_drof.xml - $CIMEROOT/components/data_comps/datm/cime_config/namelist_definition_datm.xml - $CIMEROOT/components/data_comps/dice/cime_config/namelist_definition_dice.xml - $CIMEROOT/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml - $CIMEROOT/components/data_comps/docn/cime_config/namelist_definition_docn.xml - $CIMEROOT/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml + $CIMEROOT/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml + $CIMEROOT/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml + $CIMEROOT/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml + $CIMEROOT/src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml + $CIMEROOT/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml + $CIMEROOT/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml diff --git a/cime_config/acme/machines/Depends.cetus b/config/acme/machines/Depends.cetus similarity index 100% rename from cime_config/acme/machines/Depends.cetus rename to config/acme/machines/Depends.cetus diff --git a/cime_config/acme/machines/Depends.corip1 b/config/acme/machines/Depends.corip1 similarity index 100% rename from cime_config/acme/machines/Depends.corip1 rename to config/acme/machines/Depends.corip1 diff --git a/cime_config/acme/machines/Depends.cray b/config/acme/machines/Depends.cray similarity index 100% rename from cime_config/acme/machines/Depends.cray rename to config/acme/machines/Depends.cray diff --git a/cime_config/acme/machines/Depends.gnu b/config/acme/machines/Depends.gnu similarity index 100% rename from cime_config/acme/machines/Depends.gnu rename to config/acme/machines/Depends.gnu diff --git a/cime_config/acme/machines/Depends.intel b/config/acme/machines/Depends.intel similarity index 100% rename from cime_config/acme/machines/Depends.intel rename to config/acme/machines/Depends.intel diff --git a/cime_config/acme/machines/Depends.intel14 b/config/acme/machines/Depends.intel14 similarity index 100% rename from cime_config/acme/machines/Depends.intel14 rename to config/acme/machines/Depends.intel14 diff --git a/cime_config/acme/machines/Depends.intelmic b/config/acme/machines/Depends.intelmic similarity index 100% rename from cime_config/acme/machines/Depends.intelmic rename to config/acme/machines/Depends.intelmic diff --git a/cime_config/acme/machines/Depends.intelmic14 b/config/acme/machines/Depends.intelmic14 similarity index 100% rename from cime_config/acme/machines/Depends.intelmic14 rename to config/acme/machines/Depends.intelmic14 diff --git a/cime_config/acme/machines/Depends.mira b/config/acme/machines/Depends.mira similarity index 100% rename from cime_config/acme/machines/Depends.mira rename to config/acme/machines/Depends.mira diff --git a/cime_config/acme/machines/Depends.nag b/config/acme/machines/Depends.nag similarity index 100% rename from cime_config/acme/machines/Depends.nag rename to config/acme/machines/Depends.nag diff --git a/cime_config/acme/machines/Depends.titan.cray b/config/acme/machines/Depends.titan.cray similarity index 100% rename from cime_config/acme/machines/Depends.titan.cray rename to config/acme/machines/Depends.titan.cray diff --git a/cime_config/acme/machines/Depends.titan.intel b/config/acme/machines/Depends.titan.intel similarity index 100% rename from cime_config/acme/machines/Depends.titan.intel rename to config/acme/machines/Depends.titan.intel diff --git a/cime_config/acme/machines/Depends.titan.pgi b/config/acme/machines/Depends.titan.pgi similarity index 100% rename from cime_config/acme/machines/Depends.titan.pgi rename to config/acme/machines/Depends.titan.pgi diff --git a/cime_config/acme/machines/Depends.titan.pgiacc b/config/acme/machines/Depends.titan.pgiacc similarity index 100% rename from cime_config/acme/machines/Depends.titan.pgiacc rename to config/acme/machines/Depends.titan.pgiacc diff --git a/cime_config/acme/machines/Makefile b/config/acme/machines/Makefile similarity index 97% rename from cime_config/acme/machines/Makefile rename to config/acme/machines/Makefile index 1ecad3188a5..3a67df8f2ff 100644 --- a/cime_config/acme/machines/Makefile +++ b/config/acme/machines/Makefile @@ -427,8 +427,8 @@ ifndef CISM_LIBDIR CISM_LIBDIR=$(GLC_DIR)/lib endif -INCLDIR += -I$(INSTALL_SHAREDPATH)/include -I$(CIMEROOT)/share/csm_share/shr \ - -I$(CIMEROOT)/share/csm_share/include -I$(CIMEROOT)/share/shr_RandNum/include +INCLDIR += -I$(INSTALL_SHAREDPATH)/include -I$(CIMEROOT)/src/share/util \ + -I$(CIMEROOT)/src/share/include -I$(CIMEROOT)/src/share/RandNum/include # # Use the MCT dir for the cache for all configure calls because it is the first one # @@ -569,25 +569,25 @@ endif $(SHAREDLIBROOT)/$(SHAREDPATH)/mct/Makefile.conf: @echo "SHAREDLIBROOT |$(SHAREDLIBROOT)| SHAREDPATH |$(SHAREDPATH)|"; \ - $(CONFIG_SHELL) $(CIMEROOT)/externals/mct/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/externals/mct + $(CONFIG_SHELL) $(CIMEROOT)/src/externals/mct/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/src/externals/mct $(SHAREDLIBROOT)/$(SHAREDPATH)/mct/mpi-serial/Makefile.conf: @echo "SHAREDLIBROOT |$(SHAREDLIBROOT)| SHAREDPATH |$(SHAREDPATH)|"; \ - $(CONFIG_SHELL) $(CIMEROOT)/externals/mct/mpi-serial/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/externals/mct/mpi-serial + $(CONFIG_SHELL) $(CIMEROOT)/src/externals/mct/mpi-serial/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/src/externals/mct/mpi-serial ifeq ($(PIO_VERSION),2) # This is a pio2 library PIOLIB = $(PIO_LIBDIR)/libpiof.a $(PIO_LIBDIR)/libpioc.a PIOLIBNAME = -lpiof -lpioc - PIO_SRC_DIR = $(CIMEROOT)/externals/pio2 + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio2 else # This is a pio1 library PIOLIB = $(PIO_LIBDIR)/libpio.a PIOLIBNAME = -lpio - ifneq ("$(wildcard $(CIMEROOT)/externals/pio1/pio)", "") - PIO_SRC_DIR = $(CIMEROOT)/externals/pio1 + ifneq ("$(wildcard $(CIMEROOT)/src/externals/pio1/pio)", "") + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio1 else - PIO_SRC_DIR = $(CIMEROOT)/externals/pio1/pio + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio1/pio endif endif #endif @@ -615,7 +615,7 @@ CMAKE_OPTS += -D CMAKE_Fortran_FLAGS:STRING="$(FFLAGS) $(INCLDIR)" \ -D CMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -D GPTL_PATH:STRING=$(INSTALL_SHAREDPATH) \ -D PIO_ENABLE_TESTS:BOOL=OFF \ - -D USER_CMAKE_MODULE_PATH:STRING=$(CIMEROOT)/externals/CMake + -D USER_CMAKE_MODULE_PATH:LIST="$(CIMEROOT)/src/externals/CMake;$(CIMEROOT)/src/externals/pio2/cmake" \ # Allow for separate installations of the NetCDF C and Fortran libraries ifeq ($(NETCDF_SEPARATE), false) @@ -832,7 +832,7 @@ $(COMPLIB): $(OBJS) $(CXX) -c $(INCLDIR) $(INCS) $(CXXFLAGS) $< %.F90: %.F90.in - $(CIMEROOT)/externals/genf90/genf90.pl $< > $@ + $(CIMEROOT)/src/externals/genf90/genf90.pl $< > $@ cleanatm: $(RM) -f $(LIBROOT)/libatm.a diff --git a/cime_config/acme/machines/README b/config/acme/machines/README similarity index 100% rename from cime_config/acme/machines/README rename to config/acme/machines/README diff --git a/cime_config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml similarity index 99% rename from cime_config/acme/machines/config_batch.xml rename to config/acme/machines/config_batch.xml index d6f15b0a10f..ffb8b72bfff 100644 --- a/cime_config/acme/machines/config_batch.xml +++ b/config/acme/machines/config_batch.xml @@ -254,7 +254,7 @@ --mail-user=email@pnnl.gov --output=slurm.out --error=slurm.err - + slurm diff --git a/cime_config/acme/machines/config_batch.xsd b/config/acme/machines/config_batch.xsd similarity index 100% rename from cime_config/acme/machines/config_batch.xsd rename to config/acme/machines/config_batch.xsd diff --git a/cime_config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml similarity index 99% rename from cime_config/acme/machines/config_compilers.xml rename to config/acme/machines/config_compilers.xml index e15715f7bc3..fd75180a87a 100644 --- a/cime_config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -29,7 +29,7 @@ if $MPILIB is set to mpi-serial then FC = $SFC MPICC = $SCC MPIFC = $SFC - INC_MPI = $(CIMEROOT)/externals/mct/mpi-serial + INC_MPI = $(CIMEROOT)/src/externals/mct/mpi-serial ======================================================================== Options for including C++ code in the build @@ -595,7 +595,6 @@ for mct, etc. --host=Linux -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf -mkl -lpthread -lm - -DHAVE_PAPI -DHAVE_SLASHPROC @@ -1098,8 +1097,6 @@ for mct, etc. -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKLROOT) -lmkl_rt - - $(NETCDF_HOME) $(PNETCDFROOT) diff --git a/cime_config/acme/machines/config_lt_archive.xml b/config/acme/machines/config_lt_archive.xml similarity index 100% rename from cime_config/acme/machines/config_lt_archive.xml rename to config/acme/machines/config_lt_archive.xml diff --git a/cime_config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml similarity index 98% rename from cime_config/acme/machines/config_machines.xml rename to config/acme/machines/config_machines.xml index 22889871df3..12bc309feef 100644 --- a/cime_config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -51,6 +51,8 @@ the 'executable' tag must have arguments required for the chosen mpirun, as well as the executable name. + unit_testing: can be 'true' or 'false'. + This allows using a different mpirun command to launch unit tests --> @@ -561,7 +563,7 @@ Linux workstation at Sandia on SRN with SEMS TPL modules - (s999964|climate) + (s999964|climate|penn) wwwproxy.sandia.gov:80 acme_developer LINUX @@ -900,8 +902,8 @@ /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1 /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/include /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/lib - $ENV{NETCDFROOT}/bin:$ENV{PATH} - $ENV{NETCDFROOT}/lib:/soft/intel/15.0.1/mkl/lib/intel64:$ENV{LD_LIBRARY_PATH} + /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/bin:$ENV{PATH} + /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/lib:/soft/intel/15.0.1/mkl/lib/intel64:$ENV{LD_LIBRARY_PATH} /soft/climate/pnetcdf/1.6.1/intel-15.0.1/mvapich2-2.2a-intel-15.0 @@ -1070,29 +1072,6 @@ - - Linux workstation: Andy's at SNL - penn - acme_developer - LINUX - gnu - openmpi,mpi-serial - $ENV{HOME}/acme/scratch - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - $ENV{HOME}/acme/inputdata - $ENV{HOME}/acme/ptclmdata - $CIME_OUTPUT_ROOT/archive/$CASE - csm/$CASE - - /home/agsalin/acme/baselines - $CCSMROOT/tools/cprnc/build/cprnc - agsalin at sandia dot gov - 20 - 20 - 2 - - LLNL Linux Cluster, Linux (pgi), 12 pes/node, batch system is Moab intel, pgi @@ -1580,6 +1559,7 @@ ORNL XK6, os is CNL, 16 pes/node, batch system is PBS titan + Received node event ec_node acme_developer pgi,pgiacc,intel,cray mpich,mpi-serial @@ -1596,9 +1576,11 @@ CNL pbs 16 + acme 8 16 + 10 TRUE cli115 -D PIO_BUILD_TIMING:BOOL=ON diff --git a/cime_config/acme/machines/config_pio.xml b/config/acme/machines/config_pio.xml similarity index 100% rename from cime_config/acme/machines/config_pio.xml rename to config/acme/machines/config_pio.xml diff --git a/cime_config/acme/machines/mkDepends b/config/acme/machines/mkDepends similarity index 100% rename from cime_config/acme/machines/mkDepends rename to config/acme/machines/mkDepends diff --git a/cime_config/acme/machines/mkSrcfiles b/config/acme/machines/mkSrcfiles similarity index 100% rename from cime_config/acme/machines/mkSrcfiles rename to config/acme/machines/mkSrcfiles diff --git a/cime_config/acme/machines/syslog.anvil b/config/acme/machines/syslog.anvil similarity index 100% rename from cime_config/acme/machines/syslog.anvil rename to config/acme/machines/syslog.anvil diff --git a/cime_config/acme/machines/syslog.cetus b/config/acme/machines/syslog.cetus similarity index 100% rename from cime_config/acme/machines/syslog.cetus rename to config/acme/machines/syslog.cetus diff --git a/cime_config/acme/machines/syslog.cori-haswell b/config/acme/machines/syslog.cori-haswell similarity index 100% rename from cime_config/acme/machines/syslog.cori-haswell rename to config/acme/machines/syslog.cori-haswell diff --git a/cime_config/acme/machines/syslog.cori-knl b/config/acme/machines/syslog.cori-knl similarity index 100% rename from cime_config/acme/machines/syslog.cori-knl rename to config/acme/machines/syslog.cori-knl diff --git a/cime_config/acme/machines/syslog.edison b/config/acme/machines/syslog.edison similarity index 100% rename from cime_config/acme/machines/syslog.edison rename to config/acme/machines/syslog.edison diff --git a/cime_config/acme/machines/syslog.mira b/config/acme/machines/syslog.mira similarity index 100% rename from cime_config/acme/machines/syslog.mira rename to config/acme/machines/syslog.mira diff --git a/cime_config/acme/machines/syslog.noop b/config/acme/machines/syslog.noop similarity index 100% rename from cime_config/acme/machines/syslog.noop rename to config/acme/machines/syslog.noop diff --git a/cime_config/acme/machines/syslog.titan b/config/acme/machines/syslog.titan similarity index 100% rename from cime_config/acme/machines/syslog.titan rename to config/acme/machines/syslog.titan diff --git a/cime_config/acme/machines/template.case.run b/config/acme/machines/template.case.run similarity index 100% rename from cime_config/acme/machines/template.case.run rename to config/acme/machines/template.case.run diff --git a/cime_config/acme/machines/template.case.test b/config/acme/machines/template.case.test similarity index 100% rename from cime_config/acme/machines/template.case.test rename to config/acme/machines/template.case.test diff --git a/cime_config/acme/machines/template.lt_archive b/config/acme/machines/template.lt_archive similarity index 100% rename from cime_config/acme/machines/template.lt_archive rename to config/acme/machines/template.lt_archive diff --git a/cime_config/acme/machines/template.ocn_diagnostics b/config/acme/machines/template.ocn_diagnostics similarity index 100% rename from cime_config/acme/machines/template.ocn_diagnostics rename to config/acme/machines/template.ocn_diagnostics diff --git a/cime_config/acme/machines/template.st_archive b/config/acme/machines/template.st_archive similarity index 100% rename from cime_config/acme/machines/template.st_archive rename to config/acme/machines/template.st_archive diff --git a/cime_config/acme/machines/template.timeseries b/config/acme/machines/template.timeseries similarity index 100% rename from cime_config/acme/machines/template.timeseries rename to config/acme/machines/template.timeseries diff --git a/cime_config/acme/machines/userdefined_laptop_template/README.md b/config/acme/machines/userdefined_laptop_template/README.md similarity index 100% rename from cime_config/acme/machines/userdefined_laptop_template/README.md rename to config/acme/machines/userdefined_laptop_template/README.md diff --git a/cime_config/acme/machines/userdefined_laptop_template/config_compilers.xml b/config/acme/machines/userdefined_laptop_template/config_compilers.xml similarity index 100% rename from cime_config/acme/machines/userdefined_laptop_template/config_compilers.xml rename to config/acme/machines/userdefined_laptop_template/config_compilers.xml diff --git a/cime_config/acme/machines/userdefined_laptop_template/config_machines.xml b/config/acme/machines/userdefined_laptop_template/config_machines.xml similarity index 100% rename from cime_config/acme/machines/userdefined_laptop_template/config_machines.xml rename to config/acme/machines/userdefined_laptop_template/config_machines.xml diff --git a/cime_config/acme/machines/userdefined_laptop_template/config_pes.xml b/config/acme/machines/userdefined_laptop_template/config_pes.xml similarity index 100% rename from cime_config/acme/machines/userdefined_laptop_template/config_pes.xml rename to config/acme/machines/userdefined_laptop_template/config_pes.xml diff --git a/cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam b/config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam similarity index 100% rename from cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam rename to config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam diff --git a/cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm b/config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm similarity index 100% rename from cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm rename to config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm diff --git a/cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds b/config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds similarity index 100% rename from cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds rename to config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds diff --git a/cime_config/acme/testmods_dirs/allactive/clm/betr/xmlchange_cmnds b/config/acme/testmods_dirs/allactive/clm/betr/xmlchange_cmnds similarity index 100% rename from cime_config/acme/testmods_dirs/allactive/clm/betr/xmlchange_cmnds rename to config/acme/testmods_dirs/allactive/clm/betr/xmlchange_cmnds diff --git a/config/acme/testmods_dirs/allactive/clm/eca/user_nl_clm b/config/acme/testmods_dirs/allactive/clm/eca/user_nl_clm new file mode 100644 index 00000000000..0737fe21819 --- /dev/null +++ b/config/acme/testmods_dirs/allactive/clm/eca/user_nl_clm @@ -0,0 +1,2 @@ +finidat = '' + diff --git a/cime_config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds b/config/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands similarity index 100% rename from cime_config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds rename to config/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands diff --git a/config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds b/config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds new file mode 100644 index 00000000000..db652859278 --- /dev/null +++ b/config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds @@ -0,0 +1 @@ +./xmlchange PIO_TYPENAME=netcdf diff --git a/cime_config/cesm/config_archive.xml b/config/cesm/config_archive.xml similarity index 100% rename from cime_config/cesm/config_archive.xml rename to config/cesm/config_archive.xml diff --git a/cime_config/cesm/config_files.xml b/config/cesm/config_files.xml similarity index 65% rename from cime_config/cesm/config_files.xml rename to config/cesm/config_files.xml index ba9313504fc..5c5f8d0feb6 100644 --- a/cime_config/cesm/config_files.xml +++ b/config/cesm/config_files.xml @@ -18,7 +18,7 @@ char - $CIMEROOT/cime_config/config_headers.xml + $CIMEROOT/config/config_headers.xml case_der env_case.xml contains both header and group information for all the case env_*.xml files @@ -26,43 +26,43 @@ char - $CIMEROOT/cime_config/$MODEL/config_grids.xml + $CIMEROOT/config/$MODEL/config_grids.xml case_last env_case.xml file containing specification of all supported model grids, domains and mapping files (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_grids_v2.xsd + $CIMEROOT/config/xml_schemas/config_grids_v2.xsd char - $CIMEROOT/cime_config/$MODEL/machines/config_machines.xml + $CIMEROOT/config/$MODEL/machines/config_machines.xml case_last env_case.xml file containing machine specifications for target model primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_machines.xsd + $CIMEROOT/config/xml_schemas/config_machines.xsd char - $CIMEROOT/cime_config/$MODEL/machines/config_batch.xml + $CIMEROOT/config/$MODEL/machines/config_batch.xml case_last env_case.xml file containing batch system details for target system (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_batch.xsd + $CIMEROOT/config/xml_schemas/config_batch.xsd char - $CIMEROOT/cime_config/$MODEL/machines/config_compilers.xml + $CIMEROOT/config/$MODEL/machines/config_compilers.xml case_last env_case.xml file containing compiler specifications for target model primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_compilers_v2.xsd + $CIMEROOT/config/xml_schemas/config_compilers_v2.xsd char - $CIMEROOT/cime_config/$MODEL/machines/config_pio.xml + $CIMEROOT/config/$MODEL/machines/config_pio.xml case_last env_case.xml file containing specification of pio settings for target model possible machine, compiler, mpilib, compset and/or grid attributes (for documentation only - DO NOT EDIT) @@ -70,7 +70,7 @@ char - $CIMEROOT/cime_config/config_tests.xml + $CIMEROOT/config/config_tests.xml test env_test.xml file containing system test descriptions @@ -85,8 +85,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/config_compsets.xml - $CIMEROOT/driver_cpl/cime_config/config_compsets.xml + $SRCROOT/cime_config/config_compsets.xml + $CIMEROOT/src/drivers/mct/cime_config/config_compsets.xml $SRCROOT/components/cam/cime_config/config_compsets.xml $SRCROOT/components/cism/cime_config/config_compsets.xml $SRCROOT/components/clm/cime_config/config_compsets.xml @@ -97,15 +97,15 @@ case_last env_case.xml file containing specification of all compsets for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_compsets.xsd + $CIMEROOT/config/xml_schemas/config_compsets.xsd char unset - $CIMEROOT/cime_config/$MODEL/allactive/config_pes.xml - $CIMEROOT/driver_cpl/cime_config/config_pes.xml + $SRCROOT/cime_config/config_pes.xml + $CIMEROOT/src/drivers/mct/cime_config/config_pes.xml $SRCROOT/components/cam/cime_config/config_pes.xml $SRCROOT/components/cism/cime_config/config_pes.xml $SRCROOT/components/clm/cime_config/config_pes.xml @@ -116,21 +116,21 @@ case_last env_case.xml file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_pes.xsd + $CIMEROOT/config/xml_schemas/config_pes.xsd char - $CIMEROOT/cime_config/cesm/config_archive.xml - $CIMEROOT/driver_cpl/cime_config/config_archive.xml + $CIMEROOT/config/cesm/config_archive.xml + $CIMEROOT/src/drivers/mct/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/drof/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/datm/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dice/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dlnd/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/docn/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dwav/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/drof/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/datm/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dice/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dlnd/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/docn/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dwav/cime_config/config_archive.xml $SRCROOT/components/cam/cime_config/config_archive.xml $SRCROOT/components/cism/cime_config/config_archive.xml @@ -141,13 +141,13 @@ case_last env_case.xml file containing specification of archive files for each component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_archive.xsd + $CIMEROOT/config/xml_schemas/config_archive.xsd char - $CIMEROOT/utils/python/CIME/SystemTests + $CIMEROOT/scripts/lib/CIME/SystemTests $SRCROOT/components/clm/cime_config/SystemTests $SRCROOT/components/cam/cime_config/SystemTests $SRCROOT/components/pop/cime_config/SystemTests @@ -165,8 +165,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/testlist_allactive.xml - $CIMEROOT/driver_cpl/cime_config/testdefs/testlist_drv.xml + $SRCROOT/cime_config/testlist_allactive.xml + $CIMEROOT/src/drivers/mct/cime_config/testdefs/testlist_drv.xml $SRCROOT/components/cam/cime_config/testdefs/testlist_cam.xml $SRCROOT/components/cism/cime_config/testdefs/testlist_cism.xml $SRCROOT/components/clm/cime_config/testdefs/testlist_clm.xml @@ -178,15 +178,15 @@ case_last env_case.xml file containing specification of all system tests for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/testlist.xsd + $CIMEROOT/config/xml_schemas/testlist.xsd char unset - $CIMEROOT/cime_config/$MODEL//allactive/testmods_dirs - $CIMEROOT/driver_cpl/cime_config/testdefs/testmods_dirs + $SRCROOT/cime_config/testmods_dirs + $CIMEROOT/src/drivers/mct/cime_config/testdefs/testmods_dirs $SRCROOT/components/cam/cime_config/testdefs/testmods_dirs $SRCROOT/components/cism/cime_config/testdefs/testmods_dirs $SRCROOT/components/clm/cime_config/testdefs/testmods_dirs @@ -204,8 +204,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/usermods_dirs - $CIMEROOT/driver_cpl/cime_config/usermods_dirs + $SRCROOT/cime_config/usermods_dirs + $CIMEROOT/src/drivers/mct/cime_config/usermods_dirs $SRCROOT/components/cam/cime_config/usermods_dirs $SRCROOT/components/cism/cime_config/usermods_dirs $SRCROOT/components/clm/cime_config/usermods_dirs @@ -225,16 +225,16 @@ char unset - $CIMEROOT/driver_cpl/cime_config/namelist_definition_modelio.xml - $CIMEROOT/driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml - $CIMEROOT/driver_cpl/cime_config/namelist_definition_drv.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_modelio.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv.xml - $CIMEROOT/components/data_comps/drof/cime_config/namelist_definition_drof.xml - $CIMEROOT/components/data_comps/datm/cime_config/namelist_definition_datm.xml - $CIMEROOT/components/data_comps/dice/cime_config/namelist_definition_dice.xml - $CIMEROOT/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml - $CIMEROOT/components/data_comps/docn/cime_config/namelist_definition_docn.xml - $CIMEROOT/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml + $CIMEROOT/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml + $CIMEROOT/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml + $CIMEROOT/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml + $CIMEROOT/src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml + $CIMEROOT/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml + $CIMEROOT/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml @@ -220,30 +245,56 @@ 0.23x0.31 0.23x0.31 gx1v6 + gx1v6 + + + + 0.23x0.31 + 0.23x0.31 + gx1v7 + gx1v7 0.23x0.31 0.23x0.31 tx0.1v2 + tx0.1v2 0.47x0.63 0.47x0.63 gx1v6 + gx1v6 + + + + 0.47x0.63 + 0.47x0.63 + gx1v7 + gx1v7 0.47x0.63 0.47x0.63 tx0.1v2 + tx0.1v2 0.9x1.25 0.9x1.25 gx1v6 + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + gx1v7 + gx1v7 @@ -251,6 +302,15 @@ 0.9x1.25 gx1v6 gland4 + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + gx1v7 + gland4 + gx1v7 @@ -258,6 +318,15 @@ 0.9x1.25 gx1v6 gland10 + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + gx1v7 + gland10 + gx1v7 @@ -265,6 +334,15 @@ 0.9x1.25 gx1v6 gland20 + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + gx1v7 + gland20 + gx1v7 @@ -272,6 +350,15 @@ 0.9x1.25 gx1v6 gland5UM + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + gx1v7 + gland5UM + gx1v7 @@ -281,6 +368,20 @@ gx1v6 + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + gx1v7 + + 0.9x1.25 0.9x1.25 @@ -289,10 +390,34 @@ gx1v6 + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + gland5UM + gx1v6 + + + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + gland5UM + gx1v7 + + 1.9x2.5 1.9x2.5 gx1v6 + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + gx1v7 + gx1v7 @@ -303,13 +428,28 @@ gx1v6 - + + 1.9x2.5 + 1.9x2.5 + gx1v7 + r01 + gx1v7 + 1.9x2.5 1.9x2.5 gx1v6 gland4 + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + gx1v7 + gland4 + gx1v7 @@ -317,6 +457,15 @@ 1.9x2.5 gx1v6 gland5UM + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + gx1v7 + gland5UM + gx1v7 @@ -327,6 +476,22 @@ gx1v6 + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + gland5UM + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + gland5UM + gx1v7 + + 1.9x2.5 1.9x2.5 @@ -334,6 +499,20 @@ gx1v6 + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + gx1v7 + + 1.9x2.5 1.9x2.5 @@ -342,10 +521,27 @@ gx1v6 + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + gland10 + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + gland10 + gx1v7 + + 4x5 4x5 gx3v7 + gx3v7 @@ -355,6 +551,20 @@ gx1v6 + + 0.23x0.31 + 0.23x0.31 + 0.23x0.31 + gx1v6 + + + + 0.23x0.31 + 0.23x0.31 + 0.23x0.31 + gx1v7 + + 2.5x3.33 2.5x3.33 @@ -362,6 +572,20 @@ gx1v6 + + 2.5x3.33 + 2.5x3.33 + 2.5x3.33 + gx1v6 + + + + 2.5x3.33 + 2.5x3.33 + 2.5x3.33 + gx1v7 + + 4x5 4x5 @@ -382,12 +606,21 @@ ne16np4 ne16np4 gx3v7 + gx3v6 ne30np4 ne30np4 gx1v6 + gx1v6 + + + + ne30np4 + ne30np4 + gx1v7 + gx1v7 @@ -395,6 +628,15 @@ 1.9x2.5 gx1v6 For testing tri-grid + gx1v6 + + + + ne30np4 + 1.9x2.5 + gx1v7 + For testing tri-grid + gx1v7 @@ -402,24 +644,50 @@ 0.9x1.25 gx1v6 For testing tri-grid + gx1v6 + + + + ne30np4 + 0.9x1.25 + gx1v7 + For testing tri-grid + gx1v7 ne60np4 ne60np4 gx1v6 + gx1v6 + + + + ne60np4 + ne60np4 + gx1v7 + gx1v7 ne120np4 ne120np4 gx1v6 + gx1v6 + + + + ne120np4 + ne120np4 + gx1v7 + gx1v7 ne120np4 ne120np4 tx0.1v2 + tx0.1v2 @@ -427,12 +695,22 @@ 0.23x0.31 gx1v6 For testing high resolution tri-grid + gx1v6 + + + + ne240np4 + 0.23x0.31 + gx1v7 + For testing high resolution tri-grid + gx1v7 ne240np4 ne240np4 tx0.1v2 + tx0.1v2 @@ -449,6 +727,20 @@ gx1v6 + + ne30np4 + ne30np4 + ne30np4 + gx1v6 + + + + ne30np4 + ne30np4 + ne30np4 + gx1v7 + + ne60np4 ne60np4 @@ -456,6 +748,20 @@ gx1v6 + + ne60np4 + ne60np4 + ne60np4 + gx1v6 + + + + ne60np4 + ne60np4 + ne60np4 + gx1v7 + + ne120np4 ne120np4 @@ -463,6 +769,20 @@ gx1v6 + + ne120np4 + ne120np4 + ne120np4 + gx1v6 + + + + ne120np4 + ne120np4 + ne120np4 + gx1v7 + + ne240np4 ne240np4 @@ -470,30 +790,62 @@ gx1v6 + + ne240np4 + ne240np4 + ne240np4 + gx1v6 + + + + ne240np4 + ne240np4 + ne240np4 + gx1v7 + + T31 T31 gx3v7 + gx3v7 4x5 4x5 gx3v7 + gx3v7 1.9x2.5 1.9x2.5 gx1v6 + gx1v6 + + + + 1.9x2.5 + 1.9x2.5 + gx1v7 + gx1v7 ne30np4 ne30np4 gx1v6 + gx1v6 + + + + ne30np4 + ne30np4 + gx1v7 + gx1v7 @@ -519,8 +871,8 @@ 1 1 - domain.ocn.01col.ArcticOcean.20150824.nc - domain.ocn.01col.ArcticOcean.20150824.nc + domain.ocn.01col.ArcticOcean.20150824.nc + domain.ocn.01col.ArcticOcean.20150824.nc 01col is a single-column grid for datm and POP: @@ -592,29 +944,31 @@ 1152 768 - domain.lnd.fv0.23x0.31_gx1v6.100517.nc - domain.ocn.0.23x0.31_gx1v6_101108.nc + domain.lnd.fv0.23x0.31_gx1v6.100517.nc + domain.ocn.0.23x0.31_gx1v6_101108.nc 0.23x0.31 is FV 1/4-deg grid: 576 384 - domain.lnd.fv0.47x0.63_gx1v6.090407.nc - domain.ocn.0.47x0.63_gx1v6_090408.nc + domain.lnd.fv0.47x0.63_gx1v6.090407.nc + domain.ocn.0.47x0.63_gx1v6_090408.nc 0.47x0.63 is FV 1/2-deg grid: 288 192 - domain.lnd.fv0.9x1.25_gx1v6.090309.nc - domain.ocn.0.9x1.25_gx1v6_090403.nc + domain.lnd.fv0.9x1.25_gx1v6.090309.nc + domain.ocn.0.9x1.25_gx1v6_090403.nc + domain.lnd.fv0.9x1.25_gx1v7.151020.nc + domain.ocn.fv0.9x1.25_gx1v7.151020.nc 0.9x1.25 is FV 1-deg grid: 144 96 - domain.lnd.fv1.9x2.5_gx1v6.090206.nc - domain.ocn.1.9x2.5_gx1v6_090403.nc + domain.lnd.fv1.9x2.5_gx1v6.090206.nc + domain.ocn.1.9x2.5_gx1v6_090403.nc 1.9x2.5 is FV 2-deg grid: @@ -644,7 +998,7 @@ 1024 512 - domain.lnd.T341_gx1v6.111226.nc + domain.lnd.T341_gx1v6.111226.nc T341 is Gaussian grid: Backward compatible for very high resolution Spectral-dycore experiments @@ -660,11 +1014,15 @@ 192 96 + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_gx1v7.151008.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_gx1v6.090320.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_gx3v7.090911.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_tx1v1.090122.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_tx0.1v2_090623.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_oQU120.160325.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.T62_gx1v6.130409.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.T62_gx1v7.151008.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.T62_gx3v7.130409.nc T62 is Gaussian grid: @@ -692,29 +1050,29 @@ 48602 1 - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4_gx1v6.110905.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne30np4_gx1v6_110217.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4_gx1v6.110905.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30np4_gx1v6_110217.nc ne30np4 is Spectral Elem 1-deg grid: 194402 1 - $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4_gx1v6.120406.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne60np4_gx1v6.121113.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4_gx1v6.120406.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne60np4_gx1v6.121113.nc ne60np4 is Spectral Elem 1/2-deg grid: 777602 1 - $DIN_LOC_ROOT/share/domains/domain.lnd.ne120np4_gx1v6.110502.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne120np4_gx1v6.121113.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne120np4_gx1v6.110502.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne120np4_gx1v6.121113.nc ne120np4 is Spectral Elem 1/4-deg grid: 3110402 1 - $DIN_LOC_ROOT/share/domains/domain.lnd.ne240np4_gx1v6.111226.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ne240np4_gx1v6.111226.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne240np4_gx1v6.111226.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne240np4_gx1v6.111226.nc ne240np4 is Spectral Elem 1/8-deg grid: Experimental for very high resolution experiments @@ -726,6 +1084,13 @@ gx1v6 is displaced Greenland pole v6 1-deg grid: + + 320 384 + $DIN_LOC_ROOT/share/domains/domain.ocn.gx1v7.151020.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.gx1v7.151020.nc + gx1v7 is displaced Greenland pole 1-deg grid with Caspian as a land feature: + + 100 116 $DIN_LOC_ROOT/share/domains/domain.ocn.gx3v7.120323.nc @@ -768,7 +1133,7 @@ 28574 1 - $DIN_LOC_ROOT/share/domains/domain.ocn.oQU120.160325.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.oQU120.160325.nc oQU120 is a MPAS ocean grid that is roughly 1 degree resolution: Experimental, under development @@ -804,8 +1169,8 @@ 90 50 - $DIN_LOC_ROOT/share/domains/domain.lnd.ww3a_ww3a.120222.nc - $DIN_LOC_ROOT/share/domains/domain.ocn.ww3a_ww3a.120222.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ww3a_ww3a.120222.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ww3a_ww3a.120222.nc WW3 90 x 50 global grid For testing of the WAV model @@ -883,6 +1248,14 @@ cpl/gridmaps/gx1v6/map_gx1v6_TO_fv0.9x1.25_aave.130322.nc + + cpl/gridmaps/fv0.9x1.25/map_fv0.9x1.25_TO_gx1v7_aave.151008.nc + cpl/gridmaps/fv0.9x1.25/map_fv0.9x1.25_TO_gx1v7_blin.151008.nc + cpl/gridmaps/fv0.9x1.25/map_fv0.9x1.25_TO_gx1v7_patc.151008.nc + cpl/gridmaps/gx1v7/map_gx1v7_TO_fv0.9x1.25_aave.151008.nc + cpl/gridmaps/gx1v7/map_gx1v7_TO_fv0.9x1.25_aave.151008.nc + + cpl/cpl6/map_fv0.9x1.25_to_mp120v1_aave_da_111004.nc cpl/cpl6/map_fv0.9x1.25_to_mp120v1_aave_da_111004.nc @@ -1011,6 +1384,13 @@ cpl/gridmaps/gx1v6/map_gx1v6_TO_T62_aave.130322.nc cpl/gridmaps/gx1v6/map_gx1v6_TO_T62_aave.130322.nc + + cpl/gridmaps/T62/map_T62_TO_gx1v7_aave.151008.nc + cpl/gridmaps/T62/map_T62_TO_gx1v7_blin.151008.nc + cpl/gridmaps/T62/map_T62_TO_gx1v7_patc.151008.nc + cpl/gridmaps/gx1v7/map_gx1v7_TO_T62_aave.151008.nc + cpl/gridmaps/gx1v7/map_gx1v7_TO_T62_aave.151008.nc + cpl/cpl6/map_T62_to_tx1v1_aave_da_090122.nc cpl/cpl6/map_T62_to_tx1v1_bilin_da_090122.nc @@ -1097,7 +1477,13 @@ cpl/gridmaps/gx1v6/map_gx1v6_TO_ww3a_splice_150428.nc - + + cpl/gridmaps/ww3a/map_ww3a_TO_gx1v7_splice_170214.nc + cpl/gridmaps/gx1v7/map_gx1v7_TO_ww3a_splice_170214.nc + cpl/gridmaps/gx1v7/map_gx1v7_TO_ww3a_splice_170214.nc + + + cpl/gridmaps/T31/map_T31_TO_ww3a_bilin_131104.nc @@ -1221,16 +1607,24 @@ cpl/cpl6/map_r05_TO_g16_aave.120920.nc + + cpl/gridmaps/r05/map_r05_TO_gx1v7_aave.161012.nc + + - cpl/gridmaps/rx1/map_rx1_to_gx3v7_nn_ac_161214.nc + cpl/gridmaps/rx1/map_rx1_to_gx3v7_e1000r500_161214.nc cpl/gridmaps/rx1/map_rx1_to_gx3v7_e1000r500_161214.nc cpl/gridmaps/rx1/map_rx1_to_gx1v6_nn_ac_161213.nc cpl/gridmaps/rx1/map_rx1_to_gx1v6_e1000r300_161212.nc + + cpl/gridmaps/rx1/map_rx1_to_gx1v7_nn_ac_161213.nc + cpl/gridmaps/rx1/map_rx1_to_gx1v7_e1000r300_161213.nc + - cpl/gridmaps/rx1/map_rx1_to_tx1v1_nn_ac_161214.nc + cpl/gridmaps/rx1/map_rx1_to_tx1v1_e1000r300_161214.nc cpl/gridmaps/rx1/map_rx1_to_tx1v1_e1000r300_161214.nc @@ -1243,7 +1637,7 @@ - cpl/gridmaps/r05/map_r05_to_gx3v7_nn_ac_161214.nc + cpl/gridmaps/r05/map_r05_to_gx3v7_e1000r500_161214.nc cpl/gridmaps/r05/map_r05_to_gx3v7_e1000r500_161214.nc @@ -1255,7 +1649,7 @@ cpl/gridmaps/r05/map_r05_to_gx1v7_e1000r300_161213.nc - cpl/gridmaps/r05/map_r05_to_tx1v1_nn_ac_161214.nc + cpl/gridmaps/r05/map_r05_to_tx1v1_e1000r500_161214.nc cpl/gridmaps/r05/map_r05_to_tx1v1_e1000r500_161214.nc @@ -1274,6 +1668,11 @@ ACTIVE + + ACTIVE + + + @@ -1591,6 +1990,9 @@ cpl/gridmaps/gland4km/map_gland4km_to_gx1v6_nnsm_e1000r300_161227.nc + + cpl/gridmaps/gland4km/map_gland4km_to_gx1v7_nnsm_e1000r300_170111.nc + cpl/gridmaps/gland4km/map_gland4km_to_gx3v7_nnsm_e1000r300_161227.nc @@ -1598,6 +2000,9 @@ cpl/gridmaps/gland5km/map_gland5km_to_gx1v6_nnsm_e1000r300_150512.nc + + cpl/gridmaps/gland5km/map_gland5km_to_gx1v7_nnsm_e1000r300_161012.nc + cpl/gridmaps/gland5km/map_gland5km_to_gx3v7_nnsm_e1000r300_150514.nc diff --git a/cime_config/cesm/machines/Depends.babbageKnc b/config/cesm/machines/Depends.babbageKnc similarity index 100% rename from cime_config/cesm/machines/Depends.babbageKnc rename to config/cesm/machines/Depends.babbageKnc diff --git a/cime_config/cesm/machines/Depends.bluewaters b/config/cesm/machines/Depends.bluewaters similarity index 100% rename from cime_config/cesm/machines/Depends.bluewaters rename to config/cesm/machines/Depends.bluewaters diff --git a/cime_config/cesm/machines/Depends.corip1 b/config/cesm/machines/Depends.corip1 similarity index 100% rename from cime_config/cesm/machines/Depends.corip1 rename to config/cesm/machines/Depends.corip1 diff --git a/cime_config/cesm/machines/Depends.cray b/config/cesm/machines/Depends.cray similarity index 100% rename from cime_config/cesm/machines/Depends.cray rename to config/cesm/machines/Depends.cray diff --git a/cime_config/cesm/machines/Depends.gnu b/config/cesm/machines/Depends.gnu similarity index 100% rename from cime_config/cesm/machines/Depends.gnu rename to config/cesm/machines/Depends.gnu diff --git a/cime_config/cesm/machines/Depends.goldbach b/config/cesm/machines/Depends.goldbach similarity index 100% rename from cime_config/cesm/machines/Depends.goldbach rename to config/cesm/machines/Depends.goldbach diff --git a/cime_config/cesm/machines/Depends.intel b/config/cesm/machines/Depends.intel similarity index 100% rename from cime_config/cesm/machines/Depends.intel rename to config/cesm/machines/Depends.intel diff --git a/cime_config/cesm/machines/Depends.intel14 b/config/cesm/machines/Depends.intel14 similarity index 100% rename from cime_config/cesm/machines/Depends.intel14 rename to config/cesm/machines/Depends.intel14 diff --git a/cime_config/cesm/machines/Depends.intelmic b/config/cesm/machines/Depends.intelmic similarity index 100% rename from cime_config/cesm/machines/Depends.intelmic rename to config/cesm/machines/Depends.intelmic diff --git a/cime_config/cesm/machines/Depends.intelmic14 b/config/cesm/machines/Depends.intelmic14 similarity index 100% rename from cime_config/cesm/machines/Depends.intelmic14 rename to config/cesm/machines/Depends.intelmic14 diff --git a/cime_config/cesm/machines/Depends.mira b/config/cesm/machines/Depends.mira similarity index 100% rename from cime_config/cesm/machines/Depends.mira rename to config/cesm/machines/Depends.mira diff --git a/cime_config/cesm/machines/Depends.nag b/config/cesm/machines/Depends.nag similarity index 100% rename from cime_config/cesm/machines/Depends.nag rename to config/cesm/machines/Depends.nag diff --git a/cime_config/cesm/machines/Makefile b/config/cesm/machines/Makefile similarity index 96% rename from cime_config/cesm/machines/Makefile rename to config/cesm/machines/Makefile index e9a25c401b4..43b26121318 100644 --- a/cime_config/cesm/machines/Makefile +++ b/config/cesm/machines/Makefile @@ -361,8 +361,36 @@ ifndef CISM_LIBDIR CISM_LIBDIR=$(GLC_DIR)/lib endif -INCLDIR += -I$(INSTALL_SHAREDPATH)/include -I$(CIMEROOT)/share/csm_share/shr \ - -I$(CIMEROOT)/share/csm_share/include -I$(CIMEROOT)/share/shr_RandNum/include +INCLDIR += -I$(INSTALL_SHAREDPATH)/include -I$(CIMEROOT)/src/share/util \ + -I$(CIMEROOT)/src/share/include -I$(CIMEROOT)/src/share/RandNum/include + +ifeq ($(MODEL),driver) + INCLDIR += -I$(EXEROOT)/atm/obj -I$(EXEROOT)/ice/obj -I$(EXEROOT)/ocn/obj -I$(EXEROOT)/glc/obj -I$(EXEROOT)/rof/obj -I$(EXEROOT)/wav/obj -I$(EXEROOT)/esp/obj +# nagfor and gcc have incompatible LDFLAGS. +# nagfor requires the weird "-Wl,-Wl,," syntax. +# If done in config_compilers.xml, we break MCT. + ifeq ($(strip $(COMPILER)),nag) + ifeq ($(NETCDF_SEPARATE), false) + SLIBS += -Wl,-Wl,,-rpath=$(NETCDF_PATH)/lib + else ifeq ($(NETCDF_SEPARATE), true) + SLIBS += -Wl,-Wl,,-rpath=$(NETCDF_C_PATH)/lib + SLIBS += -Wl,-Wl,,-rpath=$(NETCDF_FORTRAN_PATH)/lib + endif + endif +else + ifeq ($(strip $(COMPILER)),nag) + ifeq ($(DEBUG), TRUE) + ifeq ($(strip $(MACH)),hobart) + # GCC needs to be able to link to + # nagfor runtime to get autoconf + # tests to work. + CFLAGS += -Wl,--as-needed,--allow-shlib-undefined + SLIBS += -L$(COMPILER_PATH)/lib/NAG_Fortran -lf60rts + endif + endif + endif +endif + # # Use the MCT dir for the cache for all configure calls because it is the first one # @@ -389,8 +417,8 @@ FFLAGS_NOOPT += $(FPPDEFS) ifeq ($(findstring -cosp,$(CAM_CONFIG_OPTS)),-cosp) -# The following is for the COSP simulator code: -COSP_LIBDIR:=$(EXEROOT)/atm/obj/cosp + # The following is for the COSP simulator code: + COSP_LIBDIR:=$(EXEROOT)/atm/obj/cosp endif ifeq ($(MODEL),cam) @@ -433,34 +461,6 @@ else ifeq ($(NETCDF_SEPARATE), true) SLIBS += -L$(LIB_NETCDF_FORTRAN) -L$(LIB_NETCDF_C) -lnetcdff -lnetcdf endif - -ifeq ($(MODEL),driver) - INCLDIR += -I$(EXEROOT)/atm/obj -I$(EXEROOT)/ice/obj -I$(EXEROOT)/ocn/obj -I$(EXEROOT)/glc/obj -I$(EXEROOT)/rof/obj -I$(EXEROOT)/wav/obj -I$(EXEROOT)/esp/obj -# nagfor and gcc have incompatible LDFLAGS. -# nagfor requires the weird "-Wl,-Wl,," syntax. -# If done in config_compilers.xml, we break MCT. - ifeq ($(strip $(COMPILER)),nag) - ifeq ($(NETCDF_SEPARATE), false) - SLIBS += -Wl,-Wl,,-rpath=$(NETCDF_PATH)/lib - else ifeq ($(NETCDF_SEPARATE), true) - SLIBS += -Wl,-Wl,,-rpath=$(NETCDF_C_PATH)/lib - SLIBS += -Wl,-Wl,,-rpath=$(NETCDF_FORTRAN_PATH)/lib - endif - endif -else - ifeq ($(strip $(COMPILER)),nag) - ifeq ($(DEBUG), TRUE) - ifeq ($(strip $(MACH)),hobart) - # GCC needs to be able to link to - # nagfor runtime to get autoconf - # tests to work. - CFLAGS += -Wl,--as-needed,--allow-shlib-undefined - SLIBS += -L$(COMPILER_PATH)/lib/NAG_Fortran -lf60rts - endif - endif - endif -endif - ifdef LIB_PNETCDF SLIBS += -L$(LIB_PNETCDF) -lpnetcdf endif @@ -517,25 +517,25 @@ endif $(SHAREDLIBROOT)/$(SHAREDPATH)/mct/Makefile.conf: @echo "SHAREDLIBROOT |$(SHAREDLIBROOT)| SHAREDPATH |$(SHAREDPATH)|"; \ - $(CONFIG_SHELL) $(CIMEROOT)/externals/mct/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/externals/mct + $(CONFIG_SHELL) $(CIMEROOT)/src/externals/mct/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/src/externals/mct $(SHAREDLIBROOT)/$(SHAREDPATH)/mct/mpi-serial/Makefile.conf: @echo "SHAREDLIBROOT |$(SHAREDLIBROOT)| SHAREDPATH |$(SHAREDPATH)|"; \ - $(CONFIG_SHELL) $(CIMEROOT)/externals/mct/mpi-serial/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/externals/mct/mpi-serial + $(CONFIG_SHELL) $(CIMEROOT)/src/externals/mct/mpi-serial/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/src/externals/mct/mpi-serial ifeq ($(PIO_VERSION),2) # This is a pio2 library PIOLIB = $(PIO_LIBDIR)/libpiof.a $(PIO_LIBDIR)/libpioc.a PIOLIBNAME = -lpiof -lpioc - PIO_SRC_DIR = $(CIMEROOT)/externals/pio2 + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio2 else # This is a pio1 library PIOLIB = $(PIO_LIBDIR)/libpio.a PIOLIBNAME = -lpio - ifneq ("$(wildcard $(CIMEROOT)/externals/pio1/pio)", "") - PIO_SRC_DIR = $(CIMEROOT)/externals/pio1 + ifneq ("$(wildcard $(CIMEROOT)/src/externals/pio1/pio)", "") + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio1 else - PIO_SRC_DIR = $(CIMEROOT)/externals/pio1/pio + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio1/pio endif endif @@ -562,7 +562,7 @@ CMAKE_OPTS += -D CMAKE_Fortran_FLAGS:STRING="$(FFLAGS) $(INCLDIR)" \ -D CMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -D GPTL_PATH:STRING=$(INSTALL_SHAREDPATH) \ -D PIO_ENABLE_TESTS:BOOL=OFF \ - -D USER_CMAKE_MODULE_PATH:STRING=$(CIMEROOT)/externals/CMake + -D USER_CMAKE_MODULE_PATH:LIST="$(CIMEROOT)/src/externals/CMake;$(CIMEROOT)/src/externals/pio2/cmake" \ # Allow for separate installations of the NetCDF C and Fortran libraries ifeq ($(NETCDF_SEPARATE), false) @@ -796,7 +796,7 @@ endif $(CXX) -c $(INCLDIR) $(INCS) $(CXXFLAGS) $< %.F90: %.F90.in - $(CIMEROOT)/externals/genf90/genf90.pl $< > $@ + $(CIMEROOT)/src/externals/genf90/genf90.pl $< > $@ cleanatm: $(RM) -f $(LIBROOT)/libatm.a diff --git a/cime_config/cesm/machines/README b/config/cesm/machines/README similarity index 100% rename from cime_config/cesm/machines/README rename to config/cesm/machines/README diff --git a/cime_config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml similarity index 100% rename from cime_config/cesm/machines/config_batch.xml rename to config/cesm/machines/config_batch.xml diff --git a/cime_config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml similarity index 98% rename from cime_config/cesm/machines/config_compilers.xml rename to config/cesm/machines/config_compilers.xml index 3fb0c6972a2..f9890da41c4 100644 --- a/cime_config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -25,7 +25,7 @@ CC = $SCC FC = $SFC MPICC = $SCC MPIFC = $SFC -INC_MPI = $(CIMEROOT)/externals/mct/mpi-serial +INC_MPI = $(CIMEROOT)/src/externals/mct/mpi-serial ======================================================================== Options for including C++ code in the build @@ -109,7 +109,7 @@ using a fortran linker. -mcmodel=medium -std=gnu99 -fopenmp - -g -Wall + -g -Wall -ffpe-trap=invalid,zero,overflow -fcheck=bounds -O @@ -276,7 +276,7 @@ using a fortran linker. - -Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/cime_config/cesm/machines/nag_mpi_argument.txt + -Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt -ieee=full -O2 @@ -291,7 +291,7 @@ using a fortran linker. -mismatch_all - -Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/cime_config/cesm/machines/nag_mpi_argument.txt + -Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt -ieee=full @@ -963,6 +963,12 @@ using a fortran linker. $ENV{PNETCDF} + + + -DNO_MPIMOD + + + @@ -982,6 +988,8 @@ using a fortran linker. -DPIO_ENABLE_LOGGING=ON + + FALSE diff --git a/cime_config/cesm/machines/config_lt_archive.xml b/config/cesm/machines/config_lt_archive.xml similarity index 100% rename from cime_config/cesm/machines/config_lt_archive.xml rename to config/cesm/machines/config_lt_archive.xml diff --git a/cime_config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml similarity index 99% rename from cime_config/cesm/machines/config_machines.xml rename to config/cesm/machines/config_machines.xml index b23fd185072..276e0cae774 100644 --- a/cime_config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -43,6 +43,8 @@ the 'executable' tag must have arguments required for the chosen mpirun, as well as the executable name. + unit_testing: can be 'true' or 'false'. + This allows using a different mpirun command to launch unit tests --> @@ -1637,6 +1639,11 @@ + + + mpirun.lsf + TARGET_PROCESSOR_LIST=AUTO_SELECT mpirun.lsf $ENV{CESMDATAROOT}/tools/bin/launch diff --git a/cime_config/cesm/machines/config_pio.xml b/config/cesm/machines/config_pio.xml similarity index 94% rename from cime_config/cesm/machines/config_pio.xml rename to config/cesm/machines/config_pio.xml index 591a47e3408..801da1efdad 100644 --- a/cime_config/cesm/machines/config_pio.xml +++ b/config/cesm/machines/config_pio.xml @@ -44,12 +44,6 @@ pnetcdf - netcdf - netcdf - netcdf - netcdf - netcdf - netcdf netcdf @@ -115,13 +109,13 @@ --> - + diff --git a/cime_config/xml_schemas/entry_id_base.xsd b/config/xml_schemas/entry_id_base.xsd similarity index 95% rename from cime_config/xml_schemas/entry_id_base.xsd rename to config/xml_schemas/entry_id_base.xsd index f7ff393e522..5553e808cf2 100644 --- a/cime_config/xml_schemas/entry_id_base.xsd +++ b/config/xml_schemas/entry_id_base.xsd @@ -12,11 +12,10 @@ - - + diff --git a/cime_config/xml_schemas/entry_id_namelist.xsd b/config/xml_schemas/entry_id_namelist.xsd similarity index 93% rename from cime_config/xml_schemas/entry_id_namelist.xsd rename to config/xml_schemas/entry_id_namelist.xsd index 5ce33386986..3d95b558c51 100644 --- a/cime_config/xml_schemas/entry_id_namelist.xsd +++ b/config/xml_schemas/entry_id_namelist.xsd @@ -10,6 +10,8 @@ + + diff --git a/config/xml_schemas/env_archive.xsd b/config/xml_schemas/env_archive.xsd new file mode 100644 index 00000000000..62f0a36aec0 --- /dev/null +++ b/config/xml_schemas/env_archive.xsd @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/config/xml_schemas/env_entry_id.xsd b/config/xml_schemas/env_entry_id.xsd new file mode 100644 index 00000000000..04bc6d0dfbe --- /dev/null +++ b/config/xml_schemas/env_entry_id.xsd @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cime_config/xml_schemas/testlist.xsd b/config/xml_schemas/testlist.xsd similarity index 100% rename from cime_config/xml_schemas/testlist.xsd rename to config/xml_schemas/testlist.xsd diff --git a/externals/pio1/timing/COPYING b/externals/pio1/timing/COPYING deleted file mode 100644 index 94a9ed024d3..00000000000 --- a/externals/pio1/timing/COPYING +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/externals/pio2/ctest/runcdash-nersc-cray.sh b/externals/pio2/ctest/runcdash-nersc-cray.sh deleted file mode 100755 index c34fe4c0aba..00000000000 --- a/externals/pio2/ctest/runcdash-nersc-cray.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh - -# Get/Generate the Dashboard Model -if [ $# -eq 0 ]; then - model=Experimental -else - model=$1 -fi -module rm PrgEnv-intel -module load craype-network-aries -module load PrgEnv-cray -module load craype-haswell -module load cray-shmem -module load cray-mpich -module swap cce cce/8.4.2 -module load git/2.6.3 -module load cmake/3.3.2 -module load cray-hdf5-parallel/1.8.14 -module load cray-netcdf-hdf5parallel/4.3.3.1 -module load cray-parallel-netcdf/1.6.1 - -export CC=cc -export FC=ftn - -export PIO_DASHBOARD_ROOT=`pwd`/dashboard -export PIO_COMPILER_ID=Cray-`$CC -V 2>&1 | cut -d' ' -f5` - -if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then - mkdir "$PIO_DASHBOARD_ROOT" -fi -cd "$PIO_DASHBOARD_ROOT" - -if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src -fi -cd src - -ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/doc/CMakeLists.txt b/externals/pio2/doc/CMakeLists.txt deleted file mode 100644 index e8d97644217..00000000000 --- a/externals/pio2/doc/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -#============================================================================== -# -# API documentation with Doxygen -# -#============================================================================== - -find_package(Doxygen) -if(DOXYGEN_FOUND) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in - ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) - add_custom_target(doc - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Generating API documentation with Doxygen" VERBATIM) -endif(DOXYGEN_FOUND) diff --git a/externals/pio2/examples/c/CMakeLists.txt b/externals/pio2/examples/c/CMakeLists.txt deleted file mode 100644 index fad292f7865..00000000000 --- a/externals/pio2/examples/c/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -###-------------------------------------------------------------------------### -### CMakeList.txt for examples using pio -###-------------------------------------------------------------------------### - -# Include PIO include and lib directories -INCLUDE_DIRECTORIES(${PIO_INCLUDE_DIRS}) -LINK_DIRECTORIES(${PIO_LIB_DIR}) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0") -# Compiler-specific compiler options -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99") -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") -endif() -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") - -SET(SRC examplePio.c) -ADD_EXECUTABLE(examplePio_c ${SRC}) -TARGET_LINK_LIBRARIES(examplePio_c pioc) - -SET(SRC example1.c) -ADD_EXECUTABLE(example1 ${SRC}) -TARGET_LINK_LIBRARIES(example1 pioc) - -SET(SRC example2.c) -ADD_EXECUTABLE(example2 ${SRC}) -TARGET_LINK_LIBRARIES(example2 pioc) - -#===== MPE ===== -find_package (MPE "2.4.8" COMPONENTS C) -if (MPE_C_FOUND) - target_include_directories (example2 - PUBLIC ${MPE_C_INCLUDE_DIRS}) - target_compile_definitions (example2 - PUBLIC HAVE_MPE) - target_link_libraries (example2 pioc lmpe mpe pthread m) -else () - target_compile_definitions (example2 - PUBLIC _NOMPE) -endif () - diff --git a/externals/pio2/src/clib/CMakeLists.txt b/externals/pio2/src/clib/CMakeLists.txt deleted file mode 100644 index 093aeac70c1..00000000000 --- a/externals/pio2/src/clib/CMakeLists.txt +++ /dev/null @@ -1,211 +0,0 @@ -cmake_minimum_required (VERSION 2.8.12) -include (CheckFunctionExists) -project (PIOC C) - -#============================================================================== -# DEFINE THE TARGET -#============================================================================== - -set (PIO_C_SRCS topology.c - pio_file.c - pioc_support.c - pio_lists.c - pioc.c - pioc_sc.c - pio_spmd.c - pio_rearrange.c - pio_darray.c - bget.c) - -set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c) - -add_library (pioc ${PIO_C_SRCS} ${PIO_GENNC_SRCS}) - -# set up include-directories -include_directories( - "${PROJECT_SOURCE_DIR}" # to find foo/foo.h - "${PROJECT_BINARY_DIR}") # to find foo/config.h - -# Include the clib source directory -target_include_directories (pioc - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) - -# System and compiler CPP directives -target_compile_definitions (pioc - PUBLIC ${CMAKE_SYSTEM_DIRECTIVE}) -target_compile_definitions (pioc - PUBLIC ${CMAKE_C_COMPILER_DIRECTIVE}) - -# Compiler-specific compiler options -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - target_compile_options (pioc - PRIVATE -std=c99) -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI") - target_compile_options (pioc - PRIVATE -c99) -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") - target_compile_options (pioc - PRIVATE -std=c99) -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") - target_compile_options (pioc - PRIVATE -std=c99) -endif() - -#============================================================================== -# DEFINE THE INSTALL -#============================================================================== - -# Library -install (TARGETS pioc DESTINATION lib) - -# Include/Header File -install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include) - -#============================================================================== -# DEFINE THE DEPENDENCIES -#============================================================================== - -#===== MPI ===== -if (PIO_USE_MPISERIAL) - find_package (MPISERIAL COMPONENTS C REQUIRED) - if (MPISERIAL_C_FOUND) - target_compile_definitions (pioc - PRIVATE MPI_SERIAL) - target_include_directories (pioc - PUBLIC ${MPISERIAL_C_INCLUDE_DIRS}) - target_link_libraries (pioc - PUBLIC ${MPISERIAL_C_LIBRARIES}) - - set (WITH_PNETCDF FALSE) - endif () -else () - find_package (MPI REQUIRED) -endif () - -#===== GPTL ===== -if (PIO_ENABLE_TIMING) - find_package (GPTL COMPONENTS C QUIET) - if (GPTL_C_FOUND) - message (STATUS "Found GPTL C: ${GPTL_C_LIBRARIES}") - target_include_directories (pioc - PUBLIC ${GPTL_C_INCLUDE_DIRS}) - target_link_libraries (pioc - PUBLIC ${GPTL_C_LIBRARIES}) - else () - message (STATUS "Using internal GPTL C library for timing") - target_link_libraries (pioc - PUBLIC gptl) - endif () - target_compile_definitions (pioc - PUBLIC TIMING) -endif () - -#===== NetCDF-C ===== -find_package (NetCDF "4.3.3" COMPONENTS C) -if (NetCDF_C_FOUND) - target_include_directories (pioc - PUBLIC ${NetCDF_C_INCLUDE_DIRS}) - target_compile_definitions (pioc - PUBLIC _NETCDF) - target_link_libraries (pioc - PUBLIC ${NetCDF_C_LIBRARIES}) - if (${NetCDF_C_HAS_PARALLEL}) - target_compile_definitions (pioc - PUBLIC _NETCDF4) - endif () -else () - target_compile_definitions (pioc - PUBLIC _NONETCDF) -endif () - -#===== PnetCDF-C ===== -if (WITH_PNETCDF) - find_package (PnetCDF "1.6" COMPONENTS C REQUIRED) -endif () -if (PnetCDF_C_FOUND) - target_include_directories (pioc - PUBLIC ${PnetCDF_C_INCLUDE_DIRS}) - target_compile_definitions (pioc - PUBLIC _PNETCDF) - target_link_libraries (pioc - PUBLIC ${PnetCDF_C_LIBRARIES}) - - # Check library for varn functions - set (CMAKE_REQUIRED_LIBRARIES ${PnetCDF_C_LIBRARY}) - check_function_exists (ncmpi_get_varn PnetCDF_C_HAS_VARN) - if (PnetCDF_C_HAS_VARN) - target_compile_definitions(pioc - PUBLIC USE_PNETCDF_VARN - PUBLIC USE_PNETCDF_VARN_ON_READ) - endif() -else () - target_compile_definitions (pioc - PUBLIC _NOPNETCDF) -endif () - -#===== Add EXTRAs ===== -target_include_directories (pioc - PUBLIC ${PIO_C_EXTRA_INCLUDE_DIRS}) -target_link_libraries (pioc - PUBLIC ${PIO_C_EXTRA_LIBRARIES}) -target_compile_options (pioc - PRIVATE ${PIO_C_EXTRA_COMPILE_OPTIONS}) -target_compile_definitions (pioc - PUBLIC ${PIO_C_EXTRA_COMPILE_DEFINITIONS}) -if (PIO_C_EXTRA_LINK_FLAGS) - set_target_properties(pioc PROPERTIES - LINK_FLAGS ${PIO_C_EXTRA_LINK_FLAGS}) -endif () - -#===== Generate missing sources from templates ===== -# If both NetCDF and PnetCDF were found, then create templates -if (PnetCDF_C_FOUND AND NetCDF_C_FOUND AND PIO_GENERATE_SOURCES_FROM_TEMPLATES) - - # Find perl - find_package (Perl REQUIRED) - if (Perl_FOUND) - add_custom_command (OUTPUT ${PIO_GENNC_SRCS} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_c_put_template.c - pio_c_put_template.c - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_c_get_template.c - pio_c_get_template.c - COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ncputgetparser.pl - ${NetCDF_C_INCLUDE_DIR} ${PnetCDF_C_INCLUDE_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_c_template.c - pio_c_template.c - COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ncparser.pl - ${NetCDF_C_INCLUDE_DIR} ${PnetCDF_C_INCLUDE_DIR} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/pio_c_put_template.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_c_get_template.c - ${CMAKE_CURRENT_SOURCE_DIR}/ncputgetparser.pl - ${CMAKE_CURRENT_SOURCE_DIR}/pio_c_template.c - ${CMAKE_CURRENT_SOURCE_DIR}/ncparser.pl) - else () - message (FATAL_ERROR "Need Perl to create PIO C source files from templates") - endif () - -# If both NetCDF and PnetCDF are NOT found, then just copy existing source files -else () - add_custom_command (OUTPUT ${PIO_GENNC_SRCS} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_put_nc.c - pio_put_nc.c - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_get_nc.c - pio_get_nc.c - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c - pio_nc.c - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc4.c - pio_nc4.c - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/pio_put_nc.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_get_nc.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc4.c) -endif () - - -#===== Check for necessities ===== -if (NOT PnetCDF_C_FOUND AND NOT NetCDF_C_FOUND) - message (FATAL_ERROR "Must have PnetCDF and/or NetCDF C libraries") -endif () diff --git a/externals/pio2/src/clib/bget.c b/externals/pio2/src/clib/bget.c deleted file mode 100644 index 752ac219e74..00000000000 --- a/externals/pio2/src/clib/bget.c +++ /dev/null @@ -1,1672 +0,0 @@ -/* - - B G E T - - Buffer allocator - - Designed and implemented in April of 1972 by John Walker, based on the - Case Algol OPRO$ algorithm implemented in 1966. - - Reimplemented in 1975 by John Walker for the Interdata 70. - Reimplemented in 1977 by John Walker for the Marinchip 9900. - Reimplemented in 1982 by Duff Kurland for the Intel 8080. - - Portable C version implemented in September of 1990 by an older, wiser - instance of the original implementor. - - Souped up and/or weighed down slightly shortly thereafter by Greg - Lutz. - - AMIX edition, including the new compaction call-back option, prepared - by John Walker in July of 1992. - - Bug in built-in test program fixed, ANSI compiler warnings eradicated, - buffer pool validator implemented, and guaranteed repeatable test - added by John Walker in October of 1995. - - This program is in the public domain. - - 1. This is the book of the generations of Adam. In the day that God - created man, in the likeness of God made he him; - 2. Male and female created he them; and blessed them, and called - their name Adam, in the day when they were created. - 3. And Adam lived an hundred and thirty years, and begat a son in - his own likeness, and after his image; and called his name Seth: - 4. And the days of Adam after he had begotten Seth were eight - hundred years: and he begat sons and daughters: - 5. And all the days that Adam lived were nine hundred and thirty - years: and he died. - 6. And Seth lived an hundred and five years, and begat Enos: - 7. And Seth lived after he begat Enos eight hundred and seven years, - and begat sons and daughters: - 8. And all the days of Seth were nine hundred and twelve years: and - he died. - 9. And Enos lived ninety years, and begat Cainan: - 10. And Enos lived after he begat Cainan eight hundred and fifteen - years, and begat sons and daughters: - 11. And all the days of Enos were nine hundred and five years: and - he died. - 12. And Cainan lived seventy years and begat Mahalaleel: - 13. And Cainan lived after he begat Mahalaleel eight hundred and - forty years, and begat sons and daughters: - 14. And all the days of Cainan were nine hundred and ten years: and - he died. - 15. And Mahalaleel lived sixty and five years, and begat Jared: - 16. And Mahalaleel lived after he begat Jared eight hundred and - thirty years, and begat sons and daughters: - 17. And all the days of Mahalaleel were eight hundred ninety and - five years: and he died. - 18. And Jared lived an hundred sixty and two years, and he begat - Enoch: - 19. And Jared lived after he begat Enoch eight hundred years, and - begat sons and daughters: - 20. And all the days of Jared were nine hundred sixty and two years: - and he died. - 21. And Enoch lived sixty and five years, and begat Methuselah: - 22. And Enoch walked with God after he begat Methuselah three - hundred years, and begat sons and daughters: - 23. And all the days of Enoch were three hundred sixty and five - years: - 24. And Enoch walked with God: and he was not; for God took him. - 25. And Methuselah lived an hundred eighty and seven years, and - begat Lamech. - 26. And Methuselah lived after he begat Lamech seven hundred eighty - and two years, and begat sons and daughters: - 27. And all the days of Methuselah were nine hundred sixty and nine - years: and he died. - 28. And Lamech lived an hundred eighty and two years, and begat a - son: - 29. And he called his name Noah, saying, This same shall comfort us - concerning our work and toil of our hands, because of the ground - which the LORD hath cursed. - 30. And Lamech lived after he begat Noah five hundred ninety and - five years, and begat sons and daughters: - 31. And all the days of Lamech were seven hundred seventy and seven - years: and he died. - 32. And Noah was five hundred years old: and Noah begat Shem, Ham, - and Japheth. - - And buffers begat buffers, and links begat links, and buffer pools - begat links to chains of buffer pools containing buffers, and lo the - buffers and links and pools of buffers and pools of links to chains of - pools of buffers were fruitful and they multiplied and the Operating - System looked down upon them and said that it was Good. - - - INTRODUCTION - ============ - - BGET is a comprehensive memory allocation package which is easily - configured to the needs of an application. BGET is efficient in - both the time needed to allocate and release buffers and in the - memory overhead required for buffer pool management. It - automatically consolidates contiguous space to minimise - fragmentation. BGET is configured by compile-time definitions, - Major options include: - - * A built-in test program to exercise BGET and - demonstrate how the various functions are used. - - * Allocation by either the "first fit" or "best fit" - method. - - * Wiping buffers at release time to catch code which - references previously released storage. - - * Built-in routines to dump individual buffers or the - entire buffer pool. - - * Retrieval of allocation and pool size statistics. - - * Quantisation of buffer sizes to a power of two to - satisfy hardware alignment constraints. - - * Automatic pool compaction, growth, and shrinkage by - means of call-backs to user defined functions. - - Applications of BGET can range from storage management in - ROM-based embedded programs to providing the framework upon which - a multitasking system incorporating garbage collection is - constructed. BGET incorporates extensive internal consistency - checking using the mechanism; all these checks can be - turned off by compiling with NDEBUG defined, yielding a version of - BGET with minimal size and maximum speed. - - The basic algorithm underlying BGET has withstood the test of - time; more than 25 years have passed since the first - implementation of this code. And yet, it is substantially more - efficient than the native allocation schemes of many operating - systems: the Macintosh and Microsoft Windows to name two, on which - programs have obtained substantial speed-ups by layering BGET as - an application level memory manager atop the underlying system's. - - BGET has been implemented on the largest mainframes and the lowest - of microprocessors. It has served as the core for multitasking - operating systems, multi-thread applications, embedded software in - data network switching processors, and a host of C programs. And - while it has accreted flexibility and additional options over the - years, it remains fast, memory efficient, portable, and easy to - integrate into your program. - - - BGET IMPLEMENTATION ASSUMPTIONS - =============================== - - BGET is written in as portable a dialect of C as possible. The - only fundamental assumption about the underlying hardware - architecture is that memory is allocated is a linear array which - can be addressed as a vector of C "char" objects. On segmented - address space architectures, this generally means that BGET should - be used to allocate storage within a single segment (although some - compilers simulate linear address spaces on segmented - architectures). On segmented architectures, then, BGET buffer - pools may not be larger than a segment, but since BGET allows any - number of separate buffer pools, there is no limit on the total - storage which can be managed, only on the largest individual - object which can be allocated. Machines with a linear address - architecture, such as the VAX, 680x0, Sparc, MIPS, or the Intel - 80386 and above in native mode, may use BGET without restriction. - - - GETTING STARTED WITH BGET - ========================= - - Although BGET can be configured in a multitude of fashions, there - are three basic ways of working with BGET. The functions - mentioned below are documented in the following section. Please - excuse the forward references which are made in the interest of - providing a roadmap to guide you to the BGET functions you're - likely to need. - - Embedded Applications - --------------------- - - Embedded applications typically have a fixed area of memory - dedicated to buffer allocation (often in a separate RAM address - space distinct from the ROM that contains the executable code). - To use BGET in such an environment, simply call bpool() with the - start address and length of the buffer pool area in RAM, then - allocate buffers with bget() and release them with brel(). - Embedded applications with very limited RAM but abundant CPU speed - may benefit by configuring BGET for BestFit allocation (which is - usually not worth it in other environments). - - Malloc() Emulation - ------------------ - - If the C library malloc() function is too slow, not present in - your development environment (for example, an a native Windows or - Macintosh program), or otherwise unsuitable, you can replace it - with BGET. Initially define a buffer pool of an appropriate size - with bpool()--usually obtained by making a call to the operating - system's low-level memory allocator. Then allocate buffers with - bget(), bgetz(), and bgetr() (the last two permit the allocation - of buffers initialised to zero and [inefficient] re-allocation of - existing buffers for compatibility with C library functions). - Release buffers by calling brel(). If a buffer allocation request - fails, obtain more storage from the underlying operating system, - add it to the buffer pool by another call to bpool(), and continue - execution. - - Automatic Storage Management - ---------------------------- - - You can use BGET as your application's native memory manager and - implement automatic storage pool expansion, contraction, and - optionally application-specific memory compaction by compiling - BGET with the BECtl variable defined, then calling bectl() and - supplying functions for storage compaction, acquisition, and - release, as well as a standard pool expansion increment. All of - these functions are optional (although it doesn't make much sense - to provide a release function without an acquisition function, - does it?). Once the call-back functions have been defined with - bectl(), you simply use bget() and brel() to allocate and release - storage as before. You can supply an initial buffer pool with - bpool() or rely on automatic allocation to acquire the entire - pool. When a call on bget() cannot be satisfied, BGET first - checks if a compaction function has been supplied. If so, it is - called (with the space required to satisfy the allocation request - and a sequence number to allow the compaction routine to be called - successively without looping). If the compaction function is able - to free any storage (it needn't know whether the storage it freed - was adequate) it should return a nonzero value, whereupon BGET - will retry the allocation request and, if it fails again, call the - compaction function again with the next-higher sequence number. - - If the compaction function returns zero, indicating failure to - free space, or no compaction function is defined, BGET next tests - whether a non-NULL allocation function was supplied to bectl(). - If so, that function is called with an argument indicating how - many bytes of additional space are required. This will be the - standard pool expansion increment supplied in the call to bectl() - unless the original bget() call requested a buffer larger than - this; buffers larger than the standard pool block can be managed - "off the books" by BGET in this mode. If the allocation function - succeeds in obtaining the storage, it returns a pointer to the new - block and BGET expands the buffer pool; if it fails, the - allocation request fails and returns NULL to the caller. If a - non-NULL release function is supplied, expansion blocks which - become totally empty are released to the global free pool by - passing their addresses to the release function. - - Equipped with appropriate allocation, release, and compaction - functions, BGET can be used as part of very sophisticated memory - management strategies, including garbage collection. (Note, - however, that BGET is *not* a garbage collector by itself, and - that developing such a system requires much additional logic and - careful design of the application's memory allocation strategy.) - - - BGET FUNCTION DESCRIPTIONS - ========================== - - Functions implemented in this file (some are enabled by certain of - the optional settings below): - - void bpool(void *buffer, bufsize len); - - Create a buffer pool of bytes, using the storage starting at - . You can call bpool() subsequently to contribute - additional storage to the overall buffer pool. - - void *bget(bufsize size); - - Allocate a buffer of bytes. The address of the buffer is - returned, or NULL if insufficient memory was available to allocate - the buffer. - - void *bgetz(bufsize size); - - Allocate a buffer of bytes and clear it to all zeroes. The - address of the buffer is returned, or NULL if insufficient memory - was available to allocate the buffer. - - void *bgetr(void *buffer, bufsize newsize); - - Reallocate a buffer previously allocated by bget(), changing its - size to and preserving all existing data. NULL is - returned if insufficient memory is available to reallocate the - buffer, in which case the original buffer remains intact. - - void brel(void *buf); - - Return the buffer , previously allocated by bget(), to the - free space pool. - - void bectl(int (*compact)(bufsize sizereq, int sequence), - void *(*acquire)(bufsize size), - void (*release)(void *buf), - bufsize pool_incr); - - Expansion control: specify functions through which the package may - compact storage (or take other appropriate action) when an - allocation request fails, and optionally automatically acquire - storage for expansion blocks when necessary, and release such - blocks when they become empty. If is non-NULL, whenever - a buffer allocation request fails, the function will be - called with arguments specifying the number of bytes (total buffer - size, including header overhead) required to satisfy the - allocation request, and a sequence number indicating the number of - consecutive calls on attempting to satisfy this - allocation request. The sequence number is 1 for the first call - on for a given allocation request, and increments on - subsequent calls, permitting the function to take - increasingly dire measures in an attempt to free up storage. If - the function returns a nonzero value, the allocation - attempt is re-tried. If returns 0 (as it must if it - isn't able to release any space or add storage to the buffer - pool), the allocation request fails, which can trigger automatic - pool expansion if the argument is non-NULL. At the time - the function is called, the state of the buffer - allocator is identical to that at the moment the allocation - request was made; consequently, the function may call - brel(), bpool(), bstats(), and/or directly manipulate the buffer - pool in any manner which would be valid were the application in - control. This does not, however, relieve the function - of the need to ensure that whatever actions it takes do not change - things underneath the application that made the allocation - request. For example, a function that released a buffer - in the process of being reallocated with bgetr() would lead to - disaster. Implementing a safe and effective mechanism - requires careful design of an application's memory architecture, - and cannot generally be easily retrofitted into existing code. - - If is non-NULL, that function will be called whenever an - allocation request fails. If the function succeeds in - allocating the requested space and returns a pointer to the new - area, allocation will proceed using the expanded buffer pool. If - cannot obtain the requested space, it should return NULL - and the entire allocation process will fail. - specifies the normal expansion block size. Providing an - function will cause subsequent bget() requests for buffers too - large to be managed in the linked-block scheme (in other words, - larger than minus the buffer overhead) to be satisfied - directly by calls to the function. Automatic release of - empty pool blocks will occur only if all pool blocks in the system - are the size given by . - - void bstats(bufsize *curalloc, bufsize *totfree, - bufsize *maxfree, long *nget, long *nrel); - - The amount of space currently allocated is stored into the - variable pointed to by . The total free space (sum of - all free blocks in the pool) is stored into the variable pointed - to by , and the size of the largest single block in the - free space pool is stored into the variable pointed to by - . The variables pointed to by and are - filled, respectively, with the number of successful (non-NULL - return) bget() calls and the number of brel() calls. - - void bstatse(bufsize *pool_incr, long *npool, - long *npget, long *nprel, - long *ndget, long *ndrel); - - Extended statistics: The expansion block size will be stored into - the variable pointed to by , or the negative thereof if - automatic expansion block releases are disabled. The number of - currently active pool blocks will be stored into the variable - pointed to by . The variables pointed to by and - will be filled with, respectively, the number of expansion - block acquisitions and releases which have occurred. The - variables pointed to by and will be filled with - the number of bget() and brel() calls, respectively, managed - through blocks directly allocated by the acquisition and release - functions. - - void bufdump(void *buf); - - The buffer pointed to by is dumped on standard output. - - void bpoold(void *pool, int dumpalloc, int dumpfree); - - All buffers in the buffer pool , previously initialised by a - call on bpool(), are listed in ascending memory address order. If - is nonzero, the contents of allocated buffers are - dumped; if is nonzero, the contents of free blocks are - dumped. - - int bpoolv(void *pool); - - The named buffer pool, previously initialised by a call on - bpool(), is validated for bad pointers, overwritten data, etc. If - compiled with NDEBUG not defined, any error generates an assertion - failure. Otherwise 1 is returned if the pool is valid, 0 if an - error is found. - - - BGET CONFIGURATION - ================== -*/ -//#define PIO_USE_MALLOC 1 -#ifdef PIO_USE_MALLOC -#include -#endif - -#define TestProg 20000 /* Generate built-in test program - if defined. The value specifies - how many buffer allocation attempts - the test program should make. */ -#undef TestProg - -#define SizeQuant 4 /* Buffer allocation size quantum: - all buffers allocated are a - multiple of this size. This - MUST be a power of two. */ - -#define BufDump 1 /* Define this symbol to enable the - bpoold() function which dumps the - buffers in a buffer pool. */ - -#define BufValid 1 /* Define this symbol to enable the - bpoolv() function for validating - a buffer pool. */ - -#define DumpData 1 /* Define this symbol to enable the - bufdump() function which allows - dumping the contents of an allocated - or free buffer. */ - -#define BufStats 1 /* Define this symbol to enable the - bstats() function which calculates - the total free space in the buffer - pool, the largest available - buffer, and the total space - currently allocated. */ - -#define FreeWipe 1 /* Wipe free buffers to a guaranteed - pattern of garbage to trip up - miscreants who attempt to use - pointers into released buffers. */ - -//#define BestFit 1 -#undef BestFit - /* Use a best fit algorithm when - searching for space for an - allocation request. This uses - memory more efficiently, but - allocation will be much slower. */ - -#define BECtl 1 /* Define this symbol to enable the - bectl() function for automatic - pool space control. */ - -#include - -#ifdef lint -#define NDEBUG /* Exits in asserts confuse lint */ -/* LINTLIBRARY */ /* Don't complain about def, no ref */ -extern char *sprintf(); /* Sun includes don't define sprintf */ -#endif - -#include -#include - -#ifdef BufDump /* BufDump implies DumpData */ -#ifndef DumpData -#define DumpData 1 -#endif -#endif - -#ifdef DumpData -#include -#endif - -/* Declare the interface, including the requested buffer size type, - bufsize. */ - -#include "bget.h" - -#define MemSize size_t /* Type for size arguments to memxxx() - functions such as memcmp(). */ - -/* Queue links */ - -struct qlinks { - struct bfhead *flink; /* Forward link */ - struct bfhead *blink; /* Backward link */ -}; - -/* Header in allocated and free buffers */ - -struct bhead { - bufsize prevfree; /* Relative link back to previous - free buffer in memory or 0 if - previous buffer is allocated. */ - bufsize bsize; /* Buffer size: positive if free, - negative if allocated. */ -}; -#define BH(p) ((struct bhead *) (p)) - -/* Header in directly allocated buffers (by acqfcn) */ - -struct bdhead { - bufsize tsize; /* Total size, including overhead */ - struct bhead bh; /* Common header */ -}; -#define BDH(p) ((struct bdhead *) (p)) - -/* Header in free buffers */ - -struct bfhead { - struct bhead bh; /* Common allocated/free header */ - struct qlinks ql; /* Links on free list */ -}; -#define BFH(p) ((struct bfhead *) (p)) - -static struct bfhead freelist = { /* List of free buffers */ - {0, 0}, - {&freelist, &freelist} -}; - - -#ifdef BufStats -static bufsize totalloc = 0; /* Total space currently allocated */ -static long numget = 0, numrel = 0; /* Number of bget() and brel() calls */ -#ifdef BECtl -static long numpblk = 0; /* Number of pool blocks */ -static long numpget = 0, numprel = 0; /* Number of block gets and rels */ -static long numdget = 0, numdrel = 0; /* Number of direct gets and rels */ -#endif /* BECtl */ -#endif /* BufStats */ - -#ifdef BECtl - -/* Automatic expansion block management functions */ - -static int (*compfcn) _((bufsize sizereq, int sequence)) = NULL; -static void *(*acqfcn) _((bufsize size)) = NULL; -static void (*relfcn) _((void *buf)) = NULL; - -static bufsize exp_incr = 0; /* Expansion block size */ -static bufsize pool_len = 0; /* 0: no bpool calls have been made - -1: not all pool blocks are - the same size - >0: (common) block size for all - bpool calls made so far - */ -#endif - -/* Minimum allocation quantum: */ - -#define QLSize (sizeof(struct qlinks)) -#define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize) - -#define V (void) /* To denote unwanted returned values */ - -/* End sentinel: value placed in bsize field of dummy block delimiting - end of pool block. The most negative number which will fit in a - bufsize, defined in a way that the compiler will accept. */ - -#define ESent ((bufsize) (-(((1L << (sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2)) - -static int maxsize=0; - -/* added for PIO so that a bpool can be freed and another allocated */ -void bpoolrelease() -{ - freelist.bh.prevfree=0; - freelist.bh.bsize=0; - freelist.ql.flink=&freelist; - freelist.ql.blink=&freelist; - -#ifdef BufStats - totalloc = 0; /* Total space currently allocated */ - numget = 0; - numrel = 0; /* Number of bget() and brel() calls */ -#ifdef BECtl - numpblk = 0; /* Number of pool blocks */ - numpget = 0; - numprel = 0; /* Number of block gets and rels */ - numdget = 0; - numdrel = 0; /* Number of direct gets and rels */ -#endif /* BECtl */ -#endif /* BufStats */ - -#ifdef BECtl -/* Automatic expansion block management functions */ - compfcn = NULL; - acqfcn = NULL; - relfcn = NULL; - exp_incr = 0; - pool_len = 0; -#endif - -} - - - -/* BGET -- Allocate a buffer. */ - -void *bget(requested_size) - bufsize requested_size; -{ - bufsize size = requested_size; - struct bfhead *b; -#ifdef BestFit - struct bfhead *best; -#endif - void *buf; -#ifdef BECtl - int compactseq = 0; -#endif - -#ifdef PIO_USE_MALLOC - // if(requested_size>maxsize){ - // maxsize=requested_size; - // printf("%s %d %d\n",__FILE__,__LINE__,maxsize); - // } - buf = malloc(requested_size); - // printf("bget allocate %ld %x\n",requested_size,buf); - return(buf); -#endif - - - if(size<=0) - print_trace(NULL); - - assert(size > 0); - - if (size < SizeQ) { /* Need at least room for the */ - size = SizeQ; /* queue links. */ - } -#ifdef SizeQuant -#if SizeQuant > 1 - size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1)); -#endif -#endif - - size += sizeof(struct bhead); /* Add overhead in allocated buffer - to size required. */ - -#ifdef BECtl - /* If a compact function was provided in the call to bectl(), wrap - a loop around the allocation process to allow compaction to - intervene in case we don't find a suitable buffer in the chain. */ - - while (1) { -#endif - b = freelist.ql.flink; -#ifdef BestFit - best = &freelist; -#endif - - - /* Scan the free list searching for the first buffer big enough - to hold the requested size buffer. */ - -#ifdef BestFit - while (b != &freelist) { - // printf("%s %d %X %X %X %ld \n",__FILE__,__LINE__,b,&freelist,best,b->bh.bsize); - if (b->bh.bsize >= size) { - if ((best == &freelist) || (b->bh.bsize < best->bh.bsize)) { - best = b; - } - } - b = b->ql.flink; /* Link to next buffer */ - } - b = best; -#endif /* BestFit */ - - while (b != &freelist) { - if ((bufsize) b->bh.bsize >= size) { - - /* Buffer is big enough to satisfy the request. Allocate it - to the caller. We must decide whether the buffer is large - enough to split into the part given to the caller and a - free buffer that remains on the free list, or whether the - entire buffer should be removed from the free list and - given to the caller in its entirety. We only split the - buffer if enough room remains for a header plus the minimum - quantum of allocation. */ - - if ((b->bh.bsize - size) > (SizeQ + (sizeof(struct bhead)))) { - struct bhead *ba, *bn; - - ba = BH(((char *) b) + (b->bh.bsize - size)); - bn = BH(((char *) ba) + size); - assert(bn->prevfree == b->bh.bsize); - /* Subtract size from length of free block. */ - b->bh.bsize -= size; - /* Link allocated buffer to the previous free buffer. */ - ba->prevfree = b->bh.bsize; - /* Plug negative size into user buffer. */ - ba->bsize = -(bufsize) size; - /* Mark buffer after this one not preceded by free block. */ - bn->prevfree = 0; - -#ifdef BufStats - totalloc += size; - numget++; /* Increment number of bget() calls */ -#endif - buf = (void *) ((((char *) ba) + sizeof(struct bhead))); - return buf; - } else { - struct bhead *ba; - - ba = BH(((char *) b) + b->bh.bsize); - assert(ba->prevfree == b->bh.bsize); - - /* The buffer isn't big enough to split. Give the whole - shebang to the caller and remove it from the free list. */ - - assert(b->ql.blink->ql.flink == b); - assert(b->ql.flink->ql.blink == b); - b->ql.blink->ql.flink = b->ql.flink; - b->ql.flink->ql.blink = b->ql.blink; - -#ifdef BufStats - totalloc += b->bh.bsize; - numget++; /* Increment number of bget() calls */ -#endif - /* Negate size to mark buffer allocated. */ - b->bh.bsize = -(b->bh.bsize); - - /* Zero the back pointer in the next buffer in memory - to indicate that this buffer is allocated. */ - ba->prevfree = 0; - - /* Give user buffer starting at queue links. */ - buf = (void *) &(b->ql); - return buf; - } - } - b = b->ql.flink; /* Link to next buffer */ - } -#ifdef BECtl - - /* We failed to find a buffer. If there's a compact function - defined, notify it of the size requested. If it returns - TRUE, try the allocation again. */ - - if ((compfcn == NULL) || (!(*compfcn)(size, ++compactseq))) { - break; - } - } - - /* No buffer available with requested size free. */ - - /* Don't give up yet -- look in the reserve supply. */ - - if (acqfcn != NULL) { - if (size > exp_incr - sizeof(struct bhead)) { - - /* Request is too large to fit in a single expansion - block. Try to satisy it by a direct buffer acquisition. */ - - struct bdhead *bdh; - - size += sizeof(struct bdhead) - sizeof(struct bhead); - if ((bdh = BDH((*acqfcn)((bufsize) size))) != NULL) { - - /* Mark the buffer special by setting the size field - of its header to zero. */ - bdh->bh.bsize = 0; - bdh->bh.prevfree = 0; - bdh->tsize = size; -#ifdef BufStats - totalloc += size; - numget++; /* Increment number of bget() calls */ - numdget++; /* Direct bget() call count */ -#endif - buf = (void *) (bdh + 1); - - /*only let this happen once */ - printf("%s %d memory request exceeds block size %d %d %x\n",__FILE__,__LINE__,size,exp_incr,buf); - exp_incr = size+sizeof(struct bhead); - - return buf; - } - - } else { - - /* Try to obtain a new expansion block */ - - void *newpool; - - if ((newpool = (*acqfcn)((bufsize) exp_incr)) != NULL) { - bpool(newpool, exp_incr); - buf = bget(requested_size); /* This can't, I say, can't - get into a loop. */ - // printf("%s %d new memory block of size %d\n",__FILE__,__LINE__,exp_incr); - return buf; - } - } - } - - /* Still no buffer available */ - -#endif /* BECtl */ - - return NULL; -} - -/* BGETZ -- Allocate a buffer and clear its contents to zero. We clear - the entire contents of the buffer to zero, not just the - region requested by the caller. */ - -void *bgetz(size) - bufsize size; -{ - char *buf = (char *) bget(size); - - if (buf != NULL) { - struct bhead *b; - bufsize rsize; - - b = BH(buf - sizeof(struct bhead)); - rsize = -(b->bsize); - if (rsize == 0) { - struct bdhead *bd; - - bd = BDH(buf - sizeof(struct bdhead)); - rsize = bd->tsize - sizeof(struct bdhead); - } else { - rsize -= sizeof(struct bhead); - } - assert(rsize >= size); - V memset(buf, 0, (MemSize) rsize); - } - return ((void *) buf); -} - -/* BGETR -- Reallocate a buffer. This is a minimal implementation, - simply in terms of brel() and bget(). It could be - enhanced to allow the buffer to grow into adjacent free - blocks and to avoid moving data unnecessarily. */ - -void *bgetr(buf, size) - void *buf; - bufsize size; -{ - void *nbuf; - bufsize osize; /* Old size of buffer */ - struct bhead *b; - -#ifdef PIO_USE_MALLOC - return(realloc(buf, size)); -#endif - if ((nbuf = bget(size)) == NULL) { /* Acquire new buffer */ - return NULL; - } - if (buf == NULL) { - return nbuf; - } - b = BH(((char *) buf) - sizeof(struct bhead)); - osize = -b->bsize; -#ifdef BECtl - if (osize == 0) { - /* Buffer acquired directly through acqfcn. */ - struct bdhead *bd; - - bd = BDH(((char *) buf) - sizeof(struct bdhead)); - osize = bd->tsize - sizeof(struct bdhead); - } else -#endif - osize -= sizeof(struct bhead); - assert(osize > 0); - V memcpy((char *) nbuf, (char *) buf, /* Copy the data */ - (MemSize) ((size < osize) ? size : osize)); - brel(buf); - return nbuf; -} - -/* BREL -- Release a buffer. */ - -void brel(buf) - void *buf; -{ - struct bfhead *b, *bn; - -#ifdef PIO_USE_MALLOC - // printf("bget free %d %x\n",__LINE__,buf); - free(buf); - return; -#endif - - - if(buf==NULL) return; /* allow for null buffer */ - - b = BFH(((char *) buf) - sizeof(struct bhead)); -#ifdef BufStats - numrel++; /* Increment number of brel() calls */ -#endif - assert(buf != NULL); - -#ifdef BECtl - if (b->bh.bsize == 0) { /* Directly-acquired buffer? */ - struct bdhead *bdh; - - bdh = BDH(((char *) buf) - sizeof(struct bdhead)); - assert(b->bh.prevfree == 0); -#ifdef BufStats - totalloc -= bdh->tsize; - assert(totalloc >= 0); - numdrel++; /* Number of direct releases */ -#endif /* BufStats */ -#ifdef FreeWipe - V memset((char *) buf, 0x55, - (MemSize) (bdh->tsize - sizeof(struct bdhead))); -#endif /* FreeWipe */ - assert(relfcn != NULL); - (*relfcn)((void *) bdh); /* Release it directly. */ - return; - } -#endif /* BECtl */ - - /* Buffer size must be negative, indicating that the buffer is - allocated. */ - - if (b->bh.bsize >= 0) { - bn = NULL; - } - assert(b->bh.bsize < 0); - - /* Back pointer in next buffer must be zero, indicating the - same thing: */ - - assert(BH((char *) b - b->bh.bsize)->prevfree == 0); - -#ifdef BufStats - totalloc += b->bh.bsize; - assert(totalloc >= 0); -#endif - - /* If the back link is nonzero, the previous buffer is free. */ - - if (b->bh.prevfree != 0) { - - /* The previous buffer is free. Consolidate this buffer with it - by adding the length of this buffer to the previous free - buffer. Note that we subtract the size in the buffer being - released, since it's negative to indicate that the buffer is - allocated. */ - - register bufsize size = b->bh.bsize; - - /* Make the previous buffer the one we're working on. */ - assert(BH((char *) b - b->bh.prevfree)->bsize == b->bh.prevfree); - b = BFH(((char *) b) - b->bh.prevfree); - b->bh.bsize -= size; - } else { - - /* The previous buffer isn't allocated. Insert this buffer - on the free list as an isolated free block. */ - - assert(freelist.ql.blink->ql.flink == &freelist); - assert(freelist.ql.flink->ql.blink == &freelist); - b->ql.flink = &freelist; - b->ql.blink = freelist.ql.blink; - freelist.ql.blink = b; - b->ql.blink->ql.flink = b; - b->bh.bsize = -b->bh.bsize; - } - - /* Now we look at the next buffer in memory, located by advancing from - the start of this buffer by its size, to see if that buffer is - free. If it is, we combine this buffer with the next one in - memory, dechaining the second buffer from the free list. */ - - bn = BFH(((char *) b) + b->bh.bsize); - if (bn->bh.bsize > 0) { - - /* The buffer is free. Remove it from the free list and add - its size to that of our buffer. */ - - assert(BH((char *) bn + bn->bh.bsize)->prevfree == bn->bh.bsize); - assert(bn->ql.blink->ql.flink == bn); - assert(bn->ql.flink->ql.blink == bn); - bn->ql.blink->ql.flink = bn->ql.flink; - bn->ql.flink->ql.blink = bn->ql.blink; - b->bh.bsize += bn->bh.bsize; - - /* Finally, advance to the buffer that follows the newly - consolidated free block. We must set its backpointer to the - head of the consolidated free block. We know the next block - must be an allocated block because the process of recombination - guarantees that two free blocks will never be contiguous in - memory. */ - - bn = BFH(((char *) b) + b->bh.bsize); - } -#ifdef FreeWipe - V memset(((char *) b) + sizeof(struct bfhead), 0x55, - (MemSize) (b->bh.bsize - sizeof(struct bfhead))); -#endif - assert(bn->bh.bsize < 0); - - /* The next buffer is allocated. Set the backpointer in it to point - to this buffer; the previous free buffer in memory. */ - - bn->bh.prevfree = b->bh.bsize; - -#ifdef BECtl - - /* If a block-release function is defined, and this free buffer - constitutes the entire block, release it. Note that pool_len - is defined in such a way that the test will fail unless all - pool blocks are the same size. */ - - if (relfcn != NULL && - ((bufsize) b->bh.bsize) == (pool_len - sizeof(struct bhead))) { - - assert(b->bh.prevfree == 0); - assert(BH((char *) b + b->bh.bsize)->bsize == ESent); - assert(BH((char *) b + b->bh.bsize)->prevfree == b->bh.bsize); - /* Unlink the buffer from the free list */ - b->ql.blink->ql.flink = b->ql.flink; - b->ql.flink->ql.blink = b->ql.blink; - // printf("%s %d calling direct release for %x\n",__FILE__,__LINE__,b); - (*relfcn)(b); - // printf("%s %d completed direct release \n",__FILE__,__LINE__); -#ifdef BufStats - numprel++; /* Nr of expansion block releases */ - numpblk--; /* Total number of blocks */ - assert(numpblk == numpget - numprel); -#endif /* BufStats */ - } -#endif /* BECtl */ -} - -#ifdef BECtl - -/* BECTL -- Establish automatic pool expansion control */ - -void bectl(compact, acquire, release, pool_incr) - int (*compact) _((bufsize sizereq, int sequence)); - void *(*acquire) _((bufsize size)); - void (*release) _((void *buf)); - bufsize pool_incr; -{ - compfcn = compact; - acqfcn = acquire; - relfcn = release; - exp_incr = pool_incr; -} -#endif - -/* BPOOL -- Add a region of memory to the buffer pool. */ - -void bpool(buf, len) - void *buf; - bufsize len; -{ - struct bfhead *b = BFH(buf); - struct bhead *bn; - -#ifdef SizeQuant - len &= ~(SizeQuant - 1); -#endif -#ifdef BECtl - if (pool_len == 0) { - pool_len = len; - } else if (len != pool_len) { - pool_len = -1; - } -#ifdef BufStats - numpget++; /* Number of block acquisitions */ - numpblk++; /* Number of blocks total */ - assert(numpblk == numpget - numprel); -#endif /* BufStats */ -#endif /* BECtl */ - - /* Since the block is initially occupied by a single free buffer, - it had better not be (much) larger than the largest buffer - whose size we can store in bhead.bsize. */ - - assert(len - sizeof(struct bhead) <= -((bufsize) ESent + 1)); - - /* Clear the backpointer at the start of the block to indicate that - there is no free block prior to this one. That blocks - recombination when the first block in memory is released. */ - - b->bh.prevfree = 0; - - /* Chain the new block to the free list. */ - - assert(freelist.ql.blink->ql.flink == &freelist); - assert(freelist.ql.flink->ql.blink == &freelist); - b->ql.flink = &freelist; - b->ql.blink = freelist.ql.blink; - freelist.ql.blink = b; - b->ql.blink->ql.flink = b; - - /* Create a dummy allocated buffer at the end of the pool. This dummy - buffer is seen when a buffer at the end of the pool is released and - blocks recombination of the last buffer with the dummy buffer at - the end. The length in the dummy buffer is set to the largest - negative number to denote the end of the pool for diagnostic - routines (this specific value is not counted on by the actual - allocation and release functions). */ - - len -= sizeof(struct bhead); - b->bh.bsize = (bufsize) len; -#ifdef FreeWipe - V memset(((char *) b) + sizeof(struct bfhead), 0x55, - (MemSize) (len - sizeof(struct bfhead))); -#endif - bn = BH(((char *) b) + len); - bn->prevfree = (bufsize) len; - /* Definition of ESent assumes two's complement! */ - assert((~0) == -1); - bn->bsize = ESent; -} - -#ifdef BufStats - -void bfreespace(bufsize *totfree, bufsize *maxfree) -{ - struct bfhead *b = freelist.ql.flink; - *totfree = 0; - *maxfree = -1; - while (b != &freelist) { - assert(b->bh.bsize > 0); - *totfree += b->bh.bsize; - if (b->bh.bsize > *maxfree) { - *maxfree = b->bh.bsize; - } - b = b->ql.flink; /* Link to next buffer */ - } -} - -/* BSTATS -- Return buffer allocation free space statistics. */ - -void bstats(curalloc, totfree, maxfree, nget, nrel) - bufsize *curalloc, *totfree, *maxfree; - long *nget, *nrel; -{ - *nget = numget; - *nrel = numrel; - *curalloc = totalloc; - bfreespace(totfree, maxfree); -} - -#ifdef BECtl - -/* BSTATSE -- Return extended statistics */ - -void bstatse(pool_incr, npool, npget, nprel, ndget, ndrel) - bufsize *pool_incr; - long *npool, *npget, *nprel, *ndget, *ndrel; -{ - *pool_incr = (pool_len < 0) ? -exp_incr : exp_incr; - *npool = numpblk; - *npget = numpget; - *nprel = numprel; - *ndget = numdget; - *ndrel = numdrel; -} -#endif /* BECtl */ -#endif /* BufStats */ - -#ifdef DumpData - -/* BUFDUMP -- Dump the data in a buffer. This is called with the user - data pointer, and backs up to the buffer header. It will - dump either a free block or an allocated one. */ - -void bufdump(buf) - void *buf; -{ - struct bfhead *b; - unsigned char *bdump; - bufsize bdlen; - - b = BFH(((char *) buf) - sizeof(struct bhead)); - assert(b->bh.bsize != 0); - if (b->bh.bsize < 0) { - bdump = (unsigned char *) buf; - bdlen = (-b->bh.bsize) - sizeof(struct bhead); - } else { - bdump = (unsigned char *) (((char *) b) + sizeof(struct bfhead)); - bdlen = b->bh.bsize - sizeof(struct bfhead); - } - - while (bdlen > 0) { - int i, dupes = 0; - bufsize l = bdlen; - char bhex[50], bascii[20]; - - if (l > 16) { - l = 16; - } - - for (i = 0; i < l; i++) { - V sprintf(bhex + i * 3, "%02X ", bdump[i]); - bascii[i] = isprint(bdump[i]) ? bdump[i] : ' '; - } - bascii[i] = 0; - V printf("%-48s %s\n", bhex, bascii); - bdump += l; - bdlen -= l; - while ((bdlen > 16) && (memcmp((char *) (bdump - 16), - (char *) bdump, 16) == 0)) { - dupes++; - bdump += 16; - bdlen -= 16; - } - if (dupes > 1) { - V printf( - " (%d lines [%d bytes] identical to above line skipped)\n", - dupes, dupes * 16); - } else if (dupes == 1) { - bdump -= 16; - bdlen += 16; - } - } -} -#endif - -#ifdef BufDump - -/* BPOOLD -- Dump a buffer pool. The buffer headers are always listed. - If DUMPALLOC is nonzero, the contents of allocated buffers - are dumped. If DUMPFREE is nonzero, free blocks are - dumped as well. If FreeWipe checking is enabled, free - blocks which have been clobbered will always be dumped. */ - -void bpoold(buf, dumpalloc, dumpfree) - void *buf; - int dumpalloc, dumpfree; -{ - struct bfhead *b = BFH(buf); - - while (b->bh.bsize != ESent) { - bufsize bs = b->bh.bsize; - - if (bs < 0) { - bs = -bs; - V printf("Allocated buffer: size %6ld bytes.\n", (long) bs); - if (dumpalloc) { - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - } - } else { - char *lerr = ""; - - assert(bs > 0); - if ((b->ql.blink->ql.flink != b) || - (b->ql.flink->ql.blink != b)) { - lerr = " (Bad free list links)"; - } - V printf("Free block: size %6ld bytes.%s\n", - (long) bs, lerr); -#ifdef FreeWipe - lerr = ((char *) b) + sizeof(struct bfhead); - if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) || - (memcmp(lerr, lerr + 1, - (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) { - V printf( - "(Contents of above free block have been overstored.)\n"); - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - } else -#endif - if (dumpfree) { - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - } - } - b = BFH(((char *) b) + bs); - } -} -#endif /* BufDump */ - -#ifdef BufValid - -/* BPOOLV -- Validate a buffer pool. If NDEBUG isn't defined, - any error generates an assertion failure. */ - -int bpoolv(buf) - void *buf; -{ - struct bfhead *b = BFH(buf); - - while (b->bh.bsize != ESent) { - bufsize bs = b->bh.bsize; - - if (bs < 0) { - bs = -bs; - } else { - char *lerr = ""; - - assert(bs > 0); - if (bs <= 0) { - return 0; - } - if ((b->ql.blink->ql.flink != b) || - (b->ql.flink->ql.blink != b)) { - V printf("Free block: size %6ld bytes. (Bad free list links)\n", - (long) bs); - assert(0); - return 0; - } -#ifdef FreeWipe - lerr = ((char *) b) + sizeof(struct bfhead); - if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) || - (memcmp(lerr, lerr + 1, - (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) { - V printf( - "(Contents of above free block have been overstored.)\n"); - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - assert(0); - return 0; - } -#endif - } - b = BFH(((char *) b) + bs); - } - return 1; -} -#endif /* BufValid */ - - /***********************\ - * * - * Built-in test program * - * * - \***********************/ - -#ifdef TestProg - -#define Repeatable 1 /* Repeatable pseudorandom sequence */ - /* If Repeatable is not defined, a - time-seeded pseudorandom sequence - is generated, exercising BGET with - a different pattern of calls on each - run. */ -#define OUR_RAND /* Use our own built-in version of - rand() to guarantee the test is - 100% repeatable. */ - -#ifdef BECtl -#define PoolSize 300000 /* Test buffer pool size */ -#else -#define PoolSize 50000 /* Test buffer pool size */ -#endif -#define ExpIncr 32768 /* Test expansion block size */ -#define CompactTries 10 /* Maximum tries at compacting */ - -#define dumpAlloc 0 /* Dump allocated buffers ? */ -#define dumpFree 0 /* Dump free buffers ? */ - -#ifndef Repeatable -extern long time(); -#endif - -extern char *malloc(); -extern int free _((char *)); - -static char *bchain = NULL; /* Our private buffer chain */ -static char *bp = NULL; /* Our initial buffer pool */ - -#include - -#ifdef OUR_RAND - -static unsigned long int next = 1; - -/* Return next random integer */ - -int rand() -{ - next = next * 1103515245L + 12345; - return (unsigned int) (next / 65536L) % 32768L; -} - -/* Set seed for random generator */ - -void srand(seed) - unsigned int seed; -{ - next = seed; -} -#endif - -/* STATS -- Edit statistics returned by bstats() or bstatse(). */ - -static void stats(when) - char *when; -{ - bufsize cural, totfree, maxfree; - long nget, nfree; -#ifdef BECtl - bufsize pincr; - long totblocks, npget, nprel, ndget, ndrel; -#endif - - bstats(&cural, &totfree, &maxfree, &nget, &nfree); - V printf( - "%s: %ld gets, %ld releases. %ld in use, %ld free, largest = %ld\n", - when, nget, nfree, (long) cural, (long) totfree, (long) maxfree); -#ifdef BECtl - bstatse(&pincr, &totblocks, &npget, &nprel, &ndget, &ndrel); - V printf( - " Blocks: size = %ld, %ld (%ld bytes) in use, %ld gets, %ld frees\n", - (long)pincr, totblocks, pincr * totblocks, npget, nprel); - V printf(" %ld direct gets, %ld direct frees\n", ndget, ndrel); -#endif /* BECtl */ -} - -#ifdef BECtl -static int protect = 0; /* Disable compaction during bgetr() */ - -/* BCOMPACT -- Compaction call-back function. */ - -static int bcompact(bsize, seq) - bufsize bsize; - int seq; -{ -#ifdef CompactTries - char *bc = bchain; - int i = rand() & 0x3; - -#ifdef COMPACTRACE - V printf("Compaction requested. %ld bytes needed, sequence %d.\n", - (long) bsize, seq); -#endif - - if (protect || (seq > CompactTries)) { -#ifdef COMPACTRACE - V printf("Compaction gave up.\n"); -#endif - return 0; - } - - /* Based on a random cast, release a random buffer in the list - of allocated buffers. */ - - while (i > 0 && bc != NULL) { - bc = *((char **) bc); - i--; - } - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - *((char **) bc) = *((char **) fb); - brel((void *) fb); - return 1; - } - } - -#ifdef COMPACTRACE - V printf("Compaction bailed out.\n"); -#endif -#endif /* CompactTries */ - return 0; -} - -/* BEXPAND -- Expand pool call-back function. */ - -static void *bexpand(size) - bufsize size; -{ - void *np = NULL; - bufsize cural, totfree, maxfree; - long nget, nfree; - - /* Don't expand beyond the total allocated size given by PoolSize. */ - - bstats(&cural, &totfree, &maxfree, &nget, &nfree); - - if (cural < PoolSize) { - np = (void *) malloc((unsigned) size); - } -#ifdef EXPTRACE - V printf("Expand pool by %ld -- %s.\n", (long) size, - np == NULL ? "failed" : "succeeded"); -#endif - return np; -} - -/* BSHRINK -- Shrink buffer pool call-back function. */ - -static void bshrink(buf) - void *buf; -{ - if (((char *) buf) == bp) { -#ifdef EXPTRACE - V printf("Initial pool released.\n"); -#endif - bp = NULL; - } -#ifdef EXPTRACE - V printf("Shrink pool.\n"); -#endif - free((char *) buf); -} - -#endif /* BECtl */ - -/* Restrict buffer requests to those large enough to contain our pointer and - small enough for the CPU architecture. */ - -static bufsize blimit(bs) - bufsize bs; -{ - if (bs < sizeof(char *)) { - bs = sizeof(char *); - } - - /* This is written out in this ugly fashion because the - cool expression in sizeof(int) that auto-configured - to any length int befuddled some compilers. */ - - if (sizeof(int) == 2) { - if (bs > 32767) { - bs = 32767; - } - } else { - if (bs > 200000) { - bs = 200000; - } - } - return bs; -} - -int main() -{ - int i; - double x; - - /* Seed the random number generator. If Repeatable is defined, we - always use the same seed. Otherwise, we seed from the clock to - shake things up from run to run. */ - -#ifdef Repeatable - V srand(1234); -#else - V srand((int) time((long *) NULL)); -#endif - - /* Compute x such that pow(x, p) ranges between 1 and 4*ExpIncr as - p ranges from 0 to ExpIncr-1, with a concentration in the lower - numbers. */ - - x = 4.0 * ExpIncr; - x = log(x); - x = exp(log(4.0 * ExpIncr) / (ExpIncr - 1.0)); - -#ifdef BECtl - bectl(bcompact, bexpand, bshrink, (bufsize) ExpIncr); - bp = malloc(ExpIncr); - assert(bp != NULL); - bpool((void *) bp, (bufsize) ExpIncr); -#else - bp = malloc(PoolSize); - assert(bp != NULL); - bpool((void *) bp, (bufsize) PoolSize); -#endif - - stats("Create pool"); - V bpoolv((void *) bp); - bpoold((void *) bp, dumpAlloc, dumpFree); - - for (i = 0; i < TestProg; i++) { - char *cb; - bufsize bs = pow(x, (double) (rand() & (ExpIncr - 1))); - - assert(bs <= (((bufsize) 4) * ExpIncr)); - bs = blimit(bs); - if (rand() & 0x400) { - cb = (char *) bgetz(bs); - } else { - cb = (char *) bget(bs); - } - if (cb == NULL) { -#ifdef EasyOut - break; -#else - char *bc = bchain; - - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - *((char **) bc) = *((char **) fb); - brel((void *) fb); - } - continue; - } -#endif - } - *((char **) cb) = (char *) bchain; - bchain = cb; - - /* Based on a random cast, release a random buffer in the list - of allocated buffers. */ - - if ((rand() & 0x10) == 0) { - char *bc = bchain; - int i = rand() & 0x3; - - while (i > 0 && bc != NULL) { - bc = *((char **) bc); - i--; - } - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - *((char **) bc) = *((char **) fb); - brel((void *) fb); - } - } - } - - /* Based on a random cast, reallocate a random buffer in the list - to a random size */ - - if ((rand() & 0x20) == 0) { - char *bc = bchain; - int i = rand() & 0x3; - - while (i > 0 && bc != NULL) { - bc = *((char **) bc); - i--; - } - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - char *newb; - - bs = pow(x, (double) (rand() & (ExpIncr - 1))); - bs = blimit(bs); -#ifdef BECtl - protect = 1; /* Protect against compaction */ -#endif - newb = (char *) bgetr((void *) fb, bs); -#ifdef BECtl - protect = 0; -#endif - if (newb != NULL) { - *((char **) bc) = newb; - } - } - } - } - } - stats("\nAfter allocation"); - if (bp != NULL) { - V bpoolv((void *) bp); - bpoold((void *) bp, dumpAlloc, dumpFree); - } - - while (bchain != NULL) { - char *buf = bchain; - - bchain = *((char **) buf); - brel((void *) buf); - } - stats("\nAfter release"); -#ifndef BECtl - if (bp != NULL) { - V bpoolv((void *) bp); - bpoold((void *) bp, dumpAlloc, dumpFree); - } -#endif - - return 0; -} -#endif diff --git a/externals/pio2/src/clib/bget.h b/externals/pio2/src/clib/bget.h deleted file mode 100644 index e9b366ea134..00000000000 --- a/externals/pio2/src/clib/bget.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - - Interface definitions for bget.c, the memory management package. - -*/ -/* in PIO we use DEBUG bget uses NDEBUG */ -//#ifndef DEBUG -//#undef NDEBUG -//#endif - -#ifndef _ -#ifdef PROTOTYPES -#define _(x) x /* If compiler knows prototypes */ -#else -#define _(x) () /* It it doesn't */ -#endif /* PROTOTYPES */ -#endif - -typedef long bufsize; -void bpool _((void *buffer, bufsize len)); -void *bget _((bufsize size)); -void *bgetz _((bufsize size)); -void *bgetr _((void *buffer, bufsize newsize)); -void brel _((void *buf)); -void bectl _((int (*compact)(bufsize sizereq, int sequence), - void *(*acquire)(bufsize size), - void (*release)(void *buf), bufsize pool_incr)); -void bstats _((bufsize *curalloc, bufsize *totfree, bufsize *maxfree, - long *nget, long *nrel)); -void bstatse _((bufsize *pool_incr, long *npool, long *npget, - long *nprel, long *ndget, long *ndrel)); -void bufdump _((void *buf)); -void bpoold _((void *pool, int dumpalloc, int dumpfree)); -int bpoolv _((void *pool)); -void bpoolrelease _(); -void bfreespace _((bufsize *maxfree, bufsize *totfree)); diff --git a/externals/pio2/src/clib/pio.h b/externals/pio2/src/clib/pio.h deleted file mode 100644 index 5b8446bf21e..00000000000 --- a/externals/pio2/src/clib/pio.h +++ /dev/null @@ -1,609 +0,0 @@ -/** - * @file - * @author Jim Edwards - * @date 2014 - * @brief Public headers for the PIO C interface. - * - * - * - * - * @see http://code.google.com/p/parallelio/ - */ - -#ifndef _PIO_H_ -#define _PIO_H_ -#include -#include -#include -#include // memcpy -#include - -#ifdef _NETCDF -#include -#ifdef _NETCDF4 -#include -#endif -#endif -#ifdef _PNETCDF -#include -#endif - -#ifndef MPI_OFFSET -/** MPI_OFFSET is an integer type of size sufficient to represent the - * size (in bytes) of the largest file supported by MPI. In some MPI - * implementations MPI_OFFSET is not properly defined. */ -#define MPI_OFFSET MPI_LONG_LONG -#endif - -/** PIO_OFFSET is an integer type of size sufficient to represent the - * size (in bytes) of the largest file supported by MPI. */ -#define PIO_OFFSET MPI_OFFSET -#define PIO_Offset MPI_Offset - -/** The maximum number of variables allowed in a netCDF file. */ -#define PIO_MAX_VARS NC_MAX_VARS - - -/** - * @brief Variable description structure - * - * The variable record is the index into the unlimited dimension in the netcdf file - * typically this is the time dimension. - * ndims is the number of dimensions on the file for this variable - * request is the id of each outstanding pnetcdf request for this variable - * nreqs is the number of outstanding pnetcdf requests for this variable - * fillbuf is a memory buffer to hold fill values for this variable (write only) - * iobuf is a memory buffer to hold (write only) -*/ -typedef struct var_desc_t -{ - int record; - int ndims; - - int *request; // used for pnetcdf iput calls - int nreqs; - void *fillbuf; - void *iobuf; - -} var_desc_t; - -/** - * @brief io region structure - * - * Each IO region is a unit of data which can be described using start and count - * arrays. Each IO task may in general have multiple io regions per variable. The - * box rearranger will have at most one io region per variable. - * -*/ -typedef struct io_region -{ - int loffset; - PIO_Offset *start; - PIO_Offset *count; - struct io_region *next; -} io_region; - -/** - * @brief io descriptor structure - * - * This structure defines the mapping for a given variable between - * compute and IO decomposition. - * -*/ -typedef struct io_desc_t -{ - int ioid; - int async_id; - int nrecvs; - int ndof; - int ndims; - int num_aiotasks; - int rearranger; - int maxregions; - bool needsfill; // Does this decomp leave holes in the field (true) or write everywhere (false) - int maxbytes; // maximum number of bytes of this iodesc before flushing - MPI_Datatype basetype; - PIO_Offset llen; - int maxiobuflen; - PIO_Offset *gsize; - - int *rfrom; - int *rcount; - int *scount; - PIO_Offset *sindex; - PIO_Offset *rindex; - - MPI_Datatype *rtype; - MPI_Datatype *stype; - int num_stypes; - int holegridsize; - int maxfillregions; - io_region *firstregion; - io_region *fillregion; - - - bool handshake; - bool isend; - int max_requests; - - MPI_Comm subset_comm; - struct io_desc_t *next; -} io_desc_t; - -/** - * @brief io system descriptor structure - * - * This structure contains the general IO subsystem data - * and MPI structure - * -*/ -typedef struct iosystem_desc_t -{ - int iosysid; - MPI_Comm union_comm; - MPI_Comm io_comm; - MPI_Comm comp_comm; - MPI_Comm intercomm; - MPI_Comm my_comm; - - /** This MPI group contains the processors involved in - * computation. It is created in PIOc_Init_Intracomm(), and freed my - * PIO_finalize(). */ - MPI_Group compgroup; - - /** This MPI group contains the processors involved in I/O. It is - * created in PIOc_Init_Intracomm(), and freed my PIOc_finalize(). */ - MPI_Group iogroup; - - int num_iotasks; - int num_comptasks; - - int union_rank; - int comp_rank; - int io_rank; - - bool iomaster; - bool compmaster; - - int ioroot; - int comproot; - int *ioranks; - - int error_handler; - int default_rearranger; - - bool async_interface; - bool ioproc; - - MPI_Info info; - struct iosystem_desc_t *next; -} iosystem_desc_t; - -/** - * @brief multi buffer - * -*/ -typedef struct wmulti_buffer -{ - int ioid; - int validvars; - int arraylen; - int *vid; - int *frame; - void *fillvalue; - void *data; - struct wmulti_buffer *next; -} wmulti_buffer; - - - -/** - * @brief io system descriptor structure - * - * This structure holds information associated with each open file - * -*/ -typedef struct file_desc_t -{ - iosystem_desc_t *iosystem; - PIO_Offset buffsize; - int fh; - int iotype; - struct var_desc_t varlist[PIO_MAX_VARS]; - int mode; - struct wmulti_buffer buffer; - struct file_desc_t *next; -} file_desc_t; - -/** - * @brief These are the supported output formats - * -*/ - -enum PIO_IOTYPE{ - PIO_IOTYPE_PNETCDF=1, //< Parallel Netcdf (parallel) - PIO_IOTYPE_NETCDF=2, //< Netcdf3 Classic format (serial) - PIO_IOTYPE_NETCDF4C=3, //< NetCDF4 (HDF5) compressed format (serial) - PIO_IOTYPE_NETCDF4P=4 //< NetCDF4 (HDF5) parallel -}; - -/** - * @brief These are the supported output data rearrangement methods - * -*/ -enum PIO_REARRANGERS{ - PIO_REARR_BOX = 1, - PIO_REARR_SUBSET = 2 -}; - -/** - * @brief These are the supported error handlers - * -*/ -enum PIO_ERROR_HANDLERS{ - PIO_INTERNAL_ERROR=(-51), //< Errors cause abort - PIO_BCAST_ERROR=(-52), //< Error codes are broadcast to all tasks - PIO_RETURN_ERROR=(-53) //< Errors are returned to caller with no internal action -}; - -#if defined( _PNETCDF) || defined(_NETCDF) -#define PIO_GLOBAL NC_GLOBAL -#define PIO_UNLIMITED NC_UNLIMITED -#define PIO_DOUBLE NC_DOUBLE -#define PIO_REAL NC_FLOAT -#define PIO_FLOAT NC_FLOAT -#define PIO_INT NC_INT -#define PIO_CHAR NC_CHAR -#define PIO_NOERR NC_NOERR -#define PIO_WRITE NC_WRITE -#define PIO_NOWRITE NC_NOWRITE -#define PIO_CLOBBER NC_CLOBBER -#define PIO_NOCLOBBER NC_NOCLOBBER -#define PIO_NOFILL NC_NOFILL -#define PIO_MAX_NAME NC_MAX_NAME -#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS -#define PIO_64BIT_OFFSET NC_64BIT_OFFSET -// NC_64BIT_DATA This is a problem - need to define directly instead of using include file -#define PIO_64BIT_DATA 0x0010 -#define PIO_EBADID NC_EBADID -#define PIO_ENFILE NC_ENFILE -#define PIO_EEXIST NC_EEXIST -#define PIO_EINVAL NC_EINVAL -#define PIO_EPERM NC_EPERM -#define PIO_ENOTINDEFINE NC_ENOTINDEFINE -#define PIO_EINDEFINE NC_EINDEFINE -#define PIO_EINVALCOORDS NC_EINVALCOORDS -#define PIO_EMAXDIMS NC_EMAXDIMS -#define PIO_ENAMEINUSE NC_ENAMEINUSE -#define PIO_ENOTATT NC_ENOTATT -#define PIO_EMAXATTS NC_EMAXATTS -#define PIO_EBADTYPE NC_EBADTYPE -#define PIO_EBADDIM NC_EBADDIM -#define PIO_EUNLIMPOS NC_EUNLIMPOS -#define PIO_EMAXVARS NC_EMAXVARS -#define PIO_ENOTVAR NC_ENOTVAR -#define PIO_EGLOBAL NC_EGLOBAL -#define PIO_ENOTNC NC_ENOTNC -#define PIO_ESTS NC_ESTS -#define PIO_EMAXNAME NC_EMAXNAME -#define PIO_EUNLIMIT NC_EUNLIMIT -#define PIO_ENORECVARS NC_ENORECVARS -#define PIO_ECHAR NC_ECHAR -#define PIO_EEDGE NC_EEDGE -#define PIO_ESTRIDE NC_ESTRIDE -#define PIO_EBADNAME NC_EBADNAME -#define PIO_ERANGE NC_ERANGE -#define PIO_ENOMEM NC_ENOMEM -#define PIO_EVARSIZE NC_EVARSIZE -#define PIO_EDIMSIZE NC_EDIMSIZE -#define PIO_ETRUNC NC_ETRUNC -#define PIO_EAXISTYPE NC_EAXISTYPE -#define PIO_EDAP NC_EDAP -#define PIO_ECURL NC_ECURL -#define PIO_EIO NC_EIO -#define PIO_ENODATA NC_ENODATA -#define PIO_EDAPSVC NC_EDAPSVC -#define PIO_EDAS NC_EDAS -#define PIO_EDDS NC_EDDS -#define PIO_EDATADDS NC_EDATADDS -#define PIO_EDAPURL NC_EDAPURL -#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT -#define PIO_ETRANSLATION NC_ETRANSLATION -#define PIO_EHDFERR NC_EHDFERR -#define PIO_ECANTREAD NC_ECANTREAD -#define PIO_ECANTWRITE NC_ECANTWRITE -#define PIO_ECANTCREATE NC_ECANTCREATE -#define PIO_EFILEMETA NC_EFILEMETA -#define PIO_EDIMMETA NC_EDIMMETA -#define PIO_EATTMETA NC_EATTMETA -#define PIO_EVARMETA NC_EVARMETA -#define PIO_ENOCOMPOUND NC_ENOCOMPOUND -#define PIO_EATTEXISTS NC_EATTEXISTS -#define PIO_ENOTNC4 NC_ENOTNC4 -#define PIO_ESTRICTNC3 NC_ESTRICTNC3 -#define PIO_ENOTNC3 NC_ENOTNC3 -#define PIO_ENOPAR NC_ENOPAR -#define PIO_EPARINIT NC_EPARINIT -#define PIO_EBADGRPID NC_EBADGRPID -#define PIO_EBADTYPID NC_EBADTYPID -#define PIO_ETYPDEFINED NC_ETYPDEFINED -#define PIO_EBADFIELD NC_EBADFIELD -#define PIO_EBADCLASS NC_EBADCLASS -#define PIO_EMAPTYPE NC_EMAPTYPE -#define PIO_ELATEFILL NC_ELATEFILL -#define PIO_ELATEDEF NC_ELATEDEF -#define PIO_EDIMSCALE NC_EDIMSCALE -#define PIO_ENOGRP NC_ENOGRP -#define PIO_ESTORAGE NC_ESTORAGE -#define PIO_EBADCHUNK NC_EBADCHUNK -#define PIO_ENOTBUILT NC_ENOTBUILT -#define PIO_EDISKLESS NC_EDISKLESS - -#define PIO_FILL_DOUBLE NC_FILL_DOUBLE -#define PIO_FILL_FLOAT NC_FILL_FLOAT -#define PIO_FILL_INT NC_FILL_INT -#define PIO_FILL_CHAR NC_FILL_CHAR - -#endif -#ifdef _PNETCDF -#define PIO_EINDEP NC_EINDEP -#else -#define PIO_EINDEP (-203) -#endif -#if defined(__cplusplus) -extern "C" { -#endif -#define PIO_EBADIOTYPE -255 -#define PIO_REQ_NULL (NC_REQ_NULL-1) -int PIOc_freedecomp(int iosysid, int ioid); -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp); -int PIOc_inq_format (int ncid, int *formatp); -int PIOc_inq_varid (int ncid, const char *name, int *varidp); -int PIOc_inq_varnatts (int ncid, int varid, int *nattsp); -int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp); -int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, - int deflate_level); -int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, - int *deflatep, int *deflate_levelp); -int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp); -int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp); -int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp); -int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value); -int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep); -int PIOc_def_var_endian(int ncid, int varid, int endian); -int PIOc_inq_var_endian(int ncid, int varid, int *endianp); -int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems, float preemption); -int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp); -int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems, - float preemption); -int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp, - float *preemptionp); -int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp); -int PIOc_inq_varname (int ncid, int varid, char *name); -int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op); -int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op); -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname); -int PIOc_del_att (int ncid, int varid, const char *name); -int PIOc_inq_natts (int ncid, int *ngattsp); -int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp); -int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip); -int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip); -int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op); -int PIOc_redef (int ncid); -int PIOc_set_fill (int ncid, int fillmode, int *old_modep); -int PIOc_enddef (int ncid); -int PIOc_rename_var (int ncid, int varid, const char *name); -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op); -int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op); -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name); -int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip); -int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip); -int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op); -int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp); -int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip); -int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip); -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op); -int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op); -int PIOc_inq_nvars (int ncid, int *nvarsp); -int PIOc_rename_dim (int ncid, int dimid, const char *name); -int PIOc_inq_varndims (int ncid, int varid, int *ndimsp); -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip); -int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp); -int PIOc_inq_dimid (int ncid, const char *name, int *idp); -int PIOc_inq_unlimdim (int ncid, int *unlimdimidp); -int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp); -int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp); -int PIOc_inq_dimname (int ncid, int dimid, char *name); -int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op); -int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip); -int PIOc_sync (int ncid); -int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op); -int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op); -int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip); -int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp); -int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp); -int PIOc_inq_ndims (int ncid, int *ndimsp); -int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep); -int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip); -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip); -int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep); -int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op); -int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip); -int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[], - const int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, - const PIO_Offset *iostart,const PIO_Offset *iocount); -int PIOc_Init_Intracomm(const MPI_Comm comp_comm, - const int num_iotasks, const int stride, - const int base, const int rearr, int *iosysidp); -int PIOc_closefile(int ncid); -int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, - const char *fname, const int mode); -int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, - const char *fname, const int mode); -int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue); - int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, const int nvars, const PIO_Offset arraylen, void *array, const int frame[], void *fillvalue[], bool flushtodisk); - -int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip); -int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) ; -int PIOc_set_blocksize(const int newblocksize); - int PIOc_readmap(const char file[], int *ndims, int *gdims[], PIO_Offset *fmaplen, PIO_Offset *map[], const MPI_Comm comm); - int PIOc_readmap_from_f90(const char file[],int *ndims, int *gdims[], PIO_Offset *maplen, PIO_Offset *map[], const int f90_comm); - int PIOc_writemap(const char file[], const int ndims, const int gdims[], PIO_Offset maplen, PIO_Offset map[], const MPI_Comm comm); - int PIOc_writemap_from_f90(const char file[], const int ndims, const int gdims[], const PIO_Offset maplen, const PIO_Offset map[], const int f90_comm); - int PIOc_deletefile(const int iosysid, const char filename[]); - int PIOc_File_is_Open(int ncid); - int PIOc_Set_File_Error_Handling(int ncid, int method); - int PIOc_advanceframe(int ncid, int varid); - int PIOc_setframe(const int ncid, const int varid,const int frame); - int PIOc_get_numiotasks(int iosysid, int *numiotasks); - int PIOc_get_iorank(int iosysid, int *iorank); - int PIOc_get_local_array_size(int ioid); - int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method); - int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]); - int PIOc_Init_Intracomm(const MPI_Comm comp_comm, - const int num_iotasks, const int stride, - const int base,const int rearr, int *iosysidp); - int PIOc_finalize(const int iosysid); - int PIOc_iam_iotask(const int iosysid, bool *ioproc); - int PIOc_iotask_rank(const int iosysid, int *iorank); - int PIOc_iosystem_is_active(const int iosysid, bool *active); - int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) ; - int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) ; - int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) ; - int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF); - int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) ; - int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) ; - int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) ; - int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array); - int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) ; - int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) ; - int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) ; - int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) ; - int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) ; - int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) ; - int PIOc_get_var_double (int ncid, int varid, double *buf) ; - int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) ; - int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) ; - int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) ; - int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) ; - int PIOc_get_var_int (int ncid, int varid, int *buf) ; - int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) ; - int PIOc_put_var_short (int ncid, int varid, const short *op) ; - int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) ; - int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) ; - int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) ; - - int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op); - int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op); - int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op); - int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op); - int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op); - int PIOc_put_var_int (int ncid, int varid, const int *op); - int PIOc_put_var_longlong (int ncid, int varid, const long long *op); - int PIOc_put_var_schar (int ncid, int varid, const signed char *op); - int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op); - int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op); - int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op); - int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op); - int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op); - int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op); - int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op); - int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op); - int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op); - int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op); - int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op); - int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op); - int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op); - int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op); - int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op); - int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op); - int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op); - int PIOc_put_var_double (int ncid, int varid, const double *op); - int PIOc_put_var_float (int ncid, int varid, const float *op); - int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op); - int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op); - int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op); - int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op); - int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op); - int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op); - int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op); - int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op); - int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op); - int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op); - int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op); - int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op); - int PIOc_put_var_text (int ncid, int varid, const char *op); - int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op); - int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op); - int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op); - int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op); - int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op); - int PIOc_put_var_long (int ncid, int varid, const long *op); - int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op); - int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf); - int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf); - int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf); - int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf); - int PIOc_get_var_text (int ncid, int varid, char *buf); - int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf); - int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf); - int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf); - int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf); - int PIOc_get_var_float (int ncid, int varid, float *buf); - int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf); - int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf); - int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf); - int PIOc_get_var_long (int ncid, int varid, long *buf); - int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf); - int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf); - int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf); - int PIOc_get_var_longlong (int ncid, int varid, long long *buf); - int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf); - int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf); - int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf); - int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf); - int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf); - int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf); - int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf); - int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf); - int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf); - int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf); - int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf); - int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf); - int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf); - int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf); - int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf); - int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf); - int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf); - int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf); - int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf); - int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf); - int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf); - int PIOc_get_var_short (int ncid, int varid, short *buf); - int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf); - int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf); - int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf); - int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf); - int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf); - int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf); - int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf); - int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf); - int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf); - int PIOc_get_var_schar (int ncid, int varid, signed char *buf); - int PIOc_iotype_available(const int iotype); - -#if defined(__cplusplus) -} -#endif - -#endif // _PIO_H_ - diff --git a/externals/pio2/src/clib/pio_darray.c b/externals/pio2/src/clib/pio_darray.c deleted file mode 100644 index 75a451cba59..00000000000 --- a/externals/pio2/src/clib/pio_darray.c +++ /dev/null @@ -1,1787 +0,0 @@ -/** @file - * - * @brief This file contains the routines that read and write - * distributed arrays in PIO. - * - * When arrays are distributed, each processor holds some of the - * array. Only by combining the distributed arrays from all processor - * can the full array be obtained. - * - * @author Jim Edwards - * @bug No Known bugs - */ - -#include -#include -#define PIO_WRITE_BUFFERING 1 - -PIO_Offset PIO_BUFFER_SIZE_LIMIT=10485760; // 10MB default limit -bufsize PIO_CNBUFFER_LIMIT=33554432; -static void *CN_bpool=NULL; -static PIO_Offset maxusage=0; -/** @brief Set the pio buffer size limit, this is the size of the data buffer on the IO nodes. - * - * - * The pio_buffer_size_limit will only apply to files opened after the setting is changed. - * @param limit the size of the buffer on the IO nodes - * @return The previous limit setting. - */ - PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) - { - PIO_Offset oldsize; - oldsize = PIO_BUFFER_SIZE_LIMIT; - if(limit>0) - PIO_BUFFER_SIZE_LIMIT=limit; - return(oldsize); - } - -/** @brief Initialize the compute buffer to size PIO_CNBUFFER_LIMIT - * - * This routine initializes the compute buffer pool if the bget memory management is used. - * @param ios the iosystem descriptor which will use the new buffer - */ - -void compute_buffer_init(iosystem_desc_t ios) -{ -#ifndef PIO_USE_MALLOC - - if(CN_bpool == NULL){ - CN_bpool = malloc( PIO_CNBUFFER_LIMIT ); - if(CN_bpool==NULL){ - char errmsg[180]; - sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); - piodie(errmsg,__FILE__,__LINE__); - } - bpool( CN_bpool, PIO_CNBUFFER_LIMIT); - if(CN_bpool==NULL){ - char errmsg[180]; - sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); - piodie(errmsg,__FILE__,__LINE__); - } - bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); - } -#endif -} - -/** @ingroup PIO_write_darray - * @brief Write a single distributed field to output. This routine is only used if aggregation is off. - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] iodesc: a pointer to the defined iodescriptor for the buffer - * @param[in] vid: the variable id to be written - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] fillvalue: the optional fillvalue to be used for missing data in this buffer - */ - int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF, void *fillvalue) - { - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ndims; - int ierr; - int i; - int msg; - int mpierr; - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize; -#ifdef TIMING - GPTLstart("PIO:write_darray_nc"); -#endif - - tdsize=0; - ierr = PIO_NOERR; - - ios = file->iosystem; - if(ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); - return PIO_EBADID; - } - vdesc = (file->varlist)+vid; - - if(vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid); - return PIO_EBADID; - } - ndims = iodesc->ndims; - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - - if(ios->ioproc){ - io_region *region; - int ncid = file->fh; - int regioncnt; - int rrcnt; - void *bufptr; - void *tmp_buf=NULL; - int tsize; - size_t start[fndims]; - size_t count[fndims]; - int buflen, j; - - PIO_Offset *startlist[iodesc->maxregions]; - PIO_Offset *countlist[iodesc->maxregions]; - MPI_Type_size(iodesc->basetype, &tsize); - - region = iodesc->firstregion; - - if(vdesc->record >= 0 && ndimsiotype == PIO_IOTYPE_PNETCDF){ - // make sure we have room in the buffer ; - flush_output_buffer(file, false, tsize*(iodesc->maxiobuflen)); - } -#endif - - rrcnt=0; - for(regioncnt=0;regioncntmaxregions;regioncnt++){ - for(i=0;iloffset); - // this is a record based multidimensional array - if(vdesc->record >= 0){ - start[0] = vdesc->record; - for(i=1;istart[i-1]; - count[i] = region->count[i-1]; - } - if(count[1]>0) - count[0] = 1; - // Non-time dependent array - }else{ - for( i=0;istart[i]; - count[i] = region->count[i]; - } - } - } - - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); - if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid,(size_t *) start,(size_t *) count, (const double *) bufptr); - } else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, (size_t *) start, (size_t *) count, (const int *) bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid, (size_t *) start, (size_t *) count, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); - } - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - { - mpierr = MPI_Type_size(iodesc->basetype, &dsize); - size_t tstart[ndims], tcount[ndims]; - if(ios->io_rank==0){ - - for(i=0;inum_aiotasks;i++){ - if(i==0){ - buflen=1; - for(j=0;jio_comm); // handshake - tell the sending task I'm ready - mpierr = MPI_Recv( &buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); - if(buflen>0){ - mpierr = MPI_Recv( tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, ios->io_comm, &status); - mpierr = MPI_Recv( tcount, ndims, MPI_OFFSET, i,2*ios->num_iotasks+i, ios->io_comm, &status); - tmp_buf = malloc(buflen * dsize); - mpierr = MPI_Recv( tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); - } - } - - if(buflen>0){ - if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, tstart, tcount, (const int *) tmp_buf); - }else if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid, tstart, tcount, (const double *) tmp_buf); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid, tstart, tcount, (const float *) tmp_buf); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); - } - if(ierr == PIO_EEDGE){ - for(i=0;iio_rank < iodesc->num_aiotasks ){ - buflen=1; - for(i=0;iio_rank,tstart[0],tstart[1],tcount[0],tcount[1],buflen,ndims,fndims); - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - mpierr = MPI_Rsend( &buflen, 1, MPI_INT, 0, 1, ios->io_comm); - if(buflen>0) { - mpierr = MPI_Rsend( tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); - } - } - break; - } - break; - #endif - #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - for( i=0,dsize=1;ibasetype); - - /* if(regioncnt==0){ - for(i=0;imaxregions;i++){ - startlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - } - } - */ - if(dsize>0){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); - startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for( i=0; imaxregions-1){ - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, - // IOBUF, iodesc->llen, iodesc->basetype); - int reqn=0; - - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - - for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ - vdesc->request[i]=NC_REQ_NULL; - } - reqn = vdesc->nreqs; - }else{ - while(vdesc->request[reqn] != NC_REQ_NULL ){ - reqn++; - } - } - - ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, - IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); - if(vdesc->request[reqn] == NC_REQ_NULL){ - vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync - } - vdesc->nreqs = reqn; - - // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); - for(i=0;iiotype,__FILE__,__LINE__); - } - if(region != NULL) - region = region->next; - } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ - } // if(ios->ioproc) - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); -#ifdef TIMING - GPTLstop("PIO:write_darray_nc"); -#endif - - return ierr; - } - -/** @brief Write a set of one or more aggregated arrays to output file - * @ingroup PIO_write_darray - * - * This routine is used if aggregation is enabled, data is already on the - * io-tasks - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written - * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc - * @param[in] basetype : the basic type of the minimal data unit - * @param[in] gsize : array of the global dimensions of the field to be written - * @param[in] maxregions : max number of blocks to be written from this iotask - * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. - * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating - * @param[in] num_aiotasks : actual number of iotasks participating - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF - */ -int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[], - const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], - const int maxregions, io_region *firstregion, const PIO_Offset llen, - const int maxiobuflen, const int num_aiotasks, - void *IOBUF, const int frame[]) - { - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ierr; - int i; - int msg; - int mpierr; - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize; - int tsize; - int ncid; - tdsize=0; - ierr = PIO_NOERR; -#ifdef TIMING - GPTLstart("PIO:write_darray_multi_nc"); -#endif - - ios = file->iosystem; - if(ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); - return PIO_EBADID; - } - vdesc = (file->varlist)+vid[0]; - ncid = file->fh; - - if(vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); - return PIO_EBADID; - } - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); - MPI_Type_size(basetype, &tsize); - - if(ios->ioproc){ - io_region *region; - int regioncnt; - int rrcnt; - void *bufptr; - int buflen, j; - size_t start[fndims]; - size_t count[fndims]; - int ndims = iodesc_ndims; - - PIO_Offset *startlist[maxregions]; - PIO_Offset *countlist[maxregions]; - - ncid = file->fh; - region = firstregion; - - - rrcnt=0; - for(regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); - for(i=0;irecord >= 0){ - for(i=fndims-ndims;istart[i-(fndims-ndims)]; - count[i] = region->count[i-(fndims-ndims)]; - } - - if(fndims>1 && ndims0){ - count[0] = 1; - start[0] = frame[0]; - }else if(fndims==ndims){ - start[0]+=vdesc->record; - } - // Non-time dependent array - }else{ - for( i=0;istart[i]; - count[i] = region->count[i]; - } - } - } - - switch(file->iotype){ -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - for(int nv=0; nvrecord >= 0 && ndimsloffset)); - } - ierr = nc_var_par_access(ncid, vid[nv], NC_COLLECTIVE); - - if(basetype == MPI_DOUBLE ||basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); - } else if(basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); - }else if(basetype == MPI_FLOAT || basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); - } - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - for( i=0,dsize=1;i0){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); - startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for( i=0; iio_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, - // IOBUF, iodesc->llen, iodesc->basetype); - - //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); - for(int nv=0; nvvarlist)+vid[nv]; - if(vdesc->record >= 0 && ndimsnreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - - for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ - vdesc->request[i]=NC_REQ_NULL; - } - reqn = vdesc->nreqs; - }else{ - while(vdesc->request[reqn] != NC_REQ_NULL){ - reqn++; - } - } - ierr = ncmpi_iput_varn(ncid, vid[nv], rrcnt, startlist, countlist, - bufptr, llen, basetype, vdesc->request+reqn); - /* - ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, - bufptr, llen, basetype, &(vdesc->request)); - */ - if(vdesc->request[reqn] == NC_REQ_NULL){ - vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync - } - vdesc->nreqs += reqn+1; - - // printf("%s %d %d %d\n",__FILE__,__LINE__,vdesc->nreqs,vdesc->request[reqn]); - } - for(i=0;iiotype,__FILE__,__LINE__); - } - if(region != NULL) - region = region->next; - } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ - } // if(ios->ioproc) - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); -#ifdef TIMING - GPTLstop("PIO:write_darray_multi_nc"); -#endif - - return ierr; - } - -/** @brief Write a set of one or more aggregated arrays to output file - * @ingroup PIO_write_darray - * - * This routine is used if aggregation is enabled, data is already on the - * io-tasks - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written - * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc - * @param[in] basetype : the basic type of the minimal data unit - * @param[in] gsize : array of the global dimensions of the field to be written - * @param[in] maxregions : max number of blocks to be written from this iotask - * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. - * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating - * @param[in] num_aiotasks : actual number of iotasks participating - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF - */ -int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], - const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], - const int maxregions, io_region *firstregion, const PIO_Offset llen, - const int maxiobuflen, const int num_aiotasks, - void *IOBUF, const int frame[]) - { - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ierr; - int i; - int msg; - int mpierr; - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize; - int tsize; - int ncid; - tdsize=0; - ierr = PIO_NOERR; -#ifdef TIMING - GPTLstart("PIO:write_darray_multi_nc_serial"); -#endif - - ios = file->iosystem; - if(ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); - return PIO_EBADID; - } - vdesc = (file->varlist)+vid[0]; - ncid = file->fh; - - if(vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); - return PIO_EBADID; - } - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); - MPI_Type_size(basetype, &tsize); - - if(ios->ioproc){ - io_region *region; - int regioncnt; - int rrcnt; - void *bufptr; - int buflen, j; - size_t tmp_start[fndims*maxregions]; - size_t tmp_count[fndims*maxregions]; - - int ndims = iodesc_ndims; - - ncid = file->fh; - region = firstregion; - - - rrcnt=0; - for(regioncnt=0;regioncntrecord >= 0){ - for(i=fndims-ndims;istart[i-(fndims-ndims)]; - tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; - } - // Non-time dependent array - }else{ - for( i=0;istart[i]; - tmp_count[i+regioncnt*fndims] = region->count[i]; - } - } - region = region->next; - } - } - if(ios->io_rank>0){ - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - MPI_Send( &llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - if(llen>0){ - MPI_Send( &maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); - MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); - MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); - // printf("%s %d %ld\n",__FILE__,__LINE__,nvars*llen); - MPI_Send( IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); - } - }else{ - size_t rlen; - int rregions; - size_t start[fndims], count[fndims]; - size_t loffset; - mpierr = MPI_Type_size(basetype, &dsize); - - for(int rtask=0; rtasknum_iotasks; rtask++){ - if(rtask>0){ - mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready - MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); - if(rlen>0){ - MPI_Recv( &rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); - MPI_Recv( tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); - MPI_Recv( tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); - // printf("%s %d %d %ld\n",__FILE__,__LINE__,rtask,nvars*rlen); - MPI_Recv( IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); - } - }else{ - rlen = llen; - rregions = maxregions; - } - if(rlen>0){ - loffset = 0; - for(regioncnt=0;regioncntrecord>=0){ - if(fndims>1 && ndims0){ - count[0] = 1; - start[0] = frame[nv]; - }else if(fndims==ndims){ - start[0]+=vdesc->record; - } - } - - - - - if(basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid[nv], start, count, (const int *) bufptr); - }else if(basetype == MPI_DOUBLE || basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid[nv], start, count, (const double *) bufptr); - }else if(basetype == MPI_FLOAT || basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid[nv], start, count, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); - } - - if(ierr != PIO_NOERR){ - for(i=0;imaxregions;regioncnt++){ - } // if(rlen>0) - } // for(int rtask=0; rtasknum_iotasks; rtask++){ - - } - } // if(ios->ioproc) - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); -#ifdef TIMING - GPTLstop("PIO:write_darray_multi_nc_serial"); -#endif - - return ierr; - } - - - -/** @brief Write one or more arrays with the same IO decomposition to the file - * @ingroup PIO_write_darray - * - */ -int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, const int nvars, const PIO_Offset arraylen, void *array, const int frame[], void *fillvalue[], bool flushtodisk) - { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - - int vsize, rlen; - int ierr; - var_desc_t *vdesc0; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - if(! (file->mode & PIO_WRITE)){ - fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); - return PIO_EPERM; - } - - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - // print_trace(NULL); - //fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - - vdesc0 = file->varlist+vid[0]; - - pioassert(nvars>0,"nvars <= 0",__FILE__,__LINE__); - - ios = file->iosystem; - // rlen = iodesc->llen*nvars; - rlen=0; - if(iodesc->llen>0){ - rlen = iodesc->maxiobuflen*nvars; - } - if(vdesc0->iobuf != NULL){ - piodie("Attempt to overwrite existing io buffer",__FILE__,__LINE__); - } - if(iodesc->rearranger>0){ - if(rlen>0){ - MPI_Type_size(iodesc->basetype, &vsize); - //printf("rlen*vsize = %ld\n",rlen*vsize); - - vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); - if(vdesc0->iobuf==NULL){ - printf("%s %d %d %ld\n",__FILE__,__LINE__,nvars,vsize*rlen); - piomemerror(*ios,(size_t) rlen*(size_t) vsize, __FILE__,__LINE__); - } - if(iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX){ - if(vsize==4){ - for(int nv=0;nv < nvars; nv++){ - for(int i=0;imaxiobuflen;i++){ - ((float *) vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((float *)fillvalue)[nv]; - } - } - }else if(vsize==8){ - for(int nv=0;nv < nvars; nv++){ - for(int i=0;imaxiobuflen;i++){ - ((double *)vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((double *)fillvalue)[nv]; - } - } - } - } - } - ierr = rearrange_comp2io(*ios, iodesc, array, vdesc0->iobuf, nvars); - }/* this is wrong, need to think about it - else{ - vdesc0->iobuf = array; - } */ - switch(file->iotype){ - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_PNETCDF: - ierr = pio_write_darray_multi_nc(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); - break; - case PIO_IOTYPE_NETCDF4C: - case PIO_IOTYPE_NETCDF: - ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); - if(vdesc0->iobuf != NULL){ - brel(vdesc0->iobuf); - vdesc0->iobuf = NULL; - } - break; - - } - - - - if(iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill && - iodesc->holegridsize>0){ - if(vdesc0->fillbuf != NULL){ - piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); - } - - vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); - //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); - if(vsize==4){ - for(int nv=0;nvholegridsize;i++){ - ((float *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((float *) fillvalue)[nv]; - } - } - }else if(vsize==8){ - for(int nv=0;nvholegridsize;i++){ - ((double *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((double *) fillvalue)[nv]; - } - } - } - switch(file->iotype){ - case PIO_IOTYPE_PNETCDF: - ierr = pio_write_darray_multi_nc(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, - iodesc->holegridsize, iodesc->num_aiotasks, - vdesc0->fillbuf, frame); - break; - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: - case PIO_IOTYPE_NETCDF: - /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, - iodesc->holegridsize, iodesc->num_aiotasks, - vdesc0->fillbuf, frame); - */ - /* if(vdesc0->fillbuf != NULL){ - printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); - brel(vdesc0->fillbuf); - vdesc0->fillbuf = NULL; - } - */ - break; - } - } - - flush_output_buffer(file, flushtodisk, 0); - - - return ierr; - - } - -/** @brief Write a distributed array to the output file. - * @ingroup PIO_write_darray - * - * This routine aggregates output on the compute nodes and only sends - * it to the IO nodes when the compute buffer is full or when a flush - * is triggered. - * - * @param[in] ncid: the ncid of the open netCDF file. - * @param[in] vid: the variable ID returned by PIOc_def_var(). - * @param[in] ioid: the I/O description ID as passed back by - * PIOc_InitDecomp(). - - * @param[in] arraylen: the length of the array to be written. This - * is the length of the distrubited array. That is, the length of - * the portion of the data that is on the processor. - - * @param[in] array: pointer to the data to be written. This is a - * pointer to the distributed portion of the array that is on this - * processor. - - * @param[in] fillvalue: pointer to the fill value to be used for - * missing data. - * - * @returns 0 for success, non-zero error code for failure. - */ -#ifdef PIO_WRITE_BUFFERING - int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) - { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - var_desc_t *vdesc; - void *bufptr; - size_t rlen; - int ierr; - MPI_Datatype vtype; - wmulti_buffer *wmb; - int tsize; - int *tptr; - void *bptr; - void *fptr; - bool recordvar; - int needsflush; - bufsize totfree, maxfree; - - ierr = PIO_NOERR; - needsflush = 0; // false - file = pio_get_file_from_id(ncid); - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - if(! (file->mode & PIO_WRITE)){ - fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); - return PIO_EPERM; - } - - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - ios = file->iosystem; - - - vdesc = (file->varlist)+vid; - if(vdesc == NULL) - return PIO_EBADID; - - if(vdesc->record<0){ - recordvar=false; - }else{ - recordvar=true; - } - if(iodesc->ndof != arraylen){ - fprintf(stderr,"ndof=%ld, arraylen=%ld\n",iodesc->ndof,arraylen); - piodie("ndof != arraylen",__FILE__,__LINE__); - } - wmb = &(file->buffer); - if(wmb->ioid == -1){ - if(recordvar){ - wmb->ioid = ioid; - }else{ - wmb->ioid = -(ioid); - } - }else{ - // separate record and non-record variables - if(recordvar){ - while(wmb->next != NULL && wmb->ioid!=ioid){ - if(wmb->next!=NULL) - wmb = wmb->next; - } -#ifdef _PNETCDF - /* flush the previous record before starting a new one. this is collective */ - // if(vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || - // (wmb->frame != NULL && vdesc->record != wmb->frame[0])){ - // needsflush = 2; // flush to disk - // } -#endif - }else{ - while(wmb->next != NULL && wmb->ioid!= -(ioid)){ - if(wmb->next!=NULL) - wmb = wmb->next; - } - } - } - if((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))){ - wmb->next = (wmulti_buffer *) bget((bufsize) sizeof(wmulti_buffer)); - if(wmb->next == NULL){ - piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__); - } - wmb=wmb->next; - wmb->next=NULL; - if(recordvar){ - wmb->ioid = ioid; - }else{ - wmb->ioid = -(ioid); - } - wmb->validvars=0; - wmb->arraylen=arraylen; - wmb->vid=NULL; - wmb->data=NULL; - wmb->frame=NULL; - wmb->fillvalue=NULL; - } - - - MPI_Type_size(iodesc->basetype, &tsize); - // At this point wmb should be pointing to a new or existing buffer - // so we can add the data - // printf("%s %d %X %d %d %d\n",__FILE__,__LINE__,wmb->data,wmb->validvars,arraylen,tsize); - // cn_buffer_report(*ios, true); - bfreespace(&totfree, &maxfree); - if(needsflush==0){ - needsflush = (maxfree <= 1.1*(1+wmb->validvars)*arraylen*tsize ); - } - MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm); - - - if(needsflush > 0 ){ - // need to flush first - // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); - cn_buffer_report(*ios, true); - - flush_buffer(ncid,wmb, needsflush==2); // if needsflush == 2 flush to disk otherwise just flush to io node - } - if(arraylen > 0){ - wmb->data = bgetr( wmb->data, (1+wmb->validvars)*arraylen*tsize); - if(wmb->data == NULL){ - piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize , __FILE__,__LINE__); - } - } - wmb->vid = (int *) bgetr( wmb->vid,sizeof(int)*( 1+wmb->validvars)); - if(wmb->vid == NULL){ - piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); - } - if(vdesc->record>=0){ - wmb->frame = (int *) bgetr( wmb->frame,sizeof(int)*( 1+wmb->validvars)); - if(wmb->frame == NULL){ - piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); - } - } - if(iodesc->needsfill){ - wmb->fillvalue = bgetr( wmb->fillvalue,tsize*( 1+wmb->validvars)); - if(wmb->fillvalue == NULL){ - piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); - } - } - - - if(iodesc->needsfill){ - if(fillvalue != NULL){ - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); - }else{ - vtype = (MPI_Datatype) iodesc->basetype; - if(vtype == MPI_INTEGER){ - int fill = PIO_FILL_INT; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if(vtype == MPI_FLOAT || vtype == MPI_REAL4){ - float fill = PIO_FILL_FLOAT; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if(vtype == MPI_DOUBLE || vtype == MPI_REAL8){ - double fill = PIO_FILL_DOUBLE; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if(vtype == MPI_CHARACTER){ - char fill = PIO_FILL_CHAR; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",vtype); - } - } - - } - - wmb->arraylen = arraylen; - wmb->vid[wmb->validvars]=vid; - bufptr = (void *)((char *) wmb->data + arraylen*tsize*wmb->validvars); - if(arraylen>0){ - memcpy(bufptr, array, arraylen*tsize); - } - /* - if(tsize==8){ - double asum=0.0; - printf("%s %d %d %d %d\n",__FILE__,__LINE__,vid,arraylen,iodesc->ndof); - for(int k=0;kvalidvars,wmb->ioid,vid,bufptr); - - if(wmb->frame!=NULL){ - wmb->frame[wmb->validvars]=vdesc->record; - } - wmb->validvars++; - - // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,wmb->validvars,iodesc->maxbytes/tsize, iodesc->ndof, iodesc->llen); - if(wmb->validvars >= iodesc->maxbytes/tsize){ - PIOc_sync(ncid); - } - - return ierr; - - } -#else - -/** @brief Write a distributed array to the output file - * @ingroup PIO_write_darray - * - * This version of the routine does not buffer, all data is communicated to the io tasks - * before the routine returns - */ - int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) - { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - void *iobuf; - size_t rlen; - int tsize; - int ierr; - MPI_Datatype vtype; - - ierr = PIO_NOERR; - - - file = pio_get_file_from_id(ncid); - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - iobuf = NULL; - - ios = file->iosystem; - - rlen = iodesc->llen; - if(iodesc->rearranger>0){ - if(rlen>0){ - MPI_Type_size(iodesc->basetype, &tsize); - // iobuf = bget(tsize*rlen); - iobuf = malloc((size_t) tsize*rlen); - if(iobuf==NULL){ - piomemerror(*ios,rlen*(size_t) tsize, __FILE__,__LINE__); - } - } - // printf(" rlen = %d %ld\n",rlen,iobuf); - - // } - - - ierr = rearrange_comp2io(*ios, iodesc, array, iobuf, 1); - - printf("%s %d ",__FILE__,__LINE__); - for(int n=0;n<4;n++) - printf(" %d ",((int *) iobuf)[n]); - printf("\n"); - - }else{ - iobuf = array; - } - switch(file->iotype){ - case PIO_IOTYPE_PNETCDF: - case PIO_IOTYPE_NETCDF: - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: - ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); - } - - if(iodesc->rearranger>0 && rlen>0) - free(iobuf); - - return ierr; - - } -#endif - -/** @brief Read an array of data from a file to the (parallel) IO library. - * @ingroup PIO_read_darray - */ -int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) -{ - int ierr=PIO_NOERR; - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ndims, fndims; - MPI_Status status; - int i; - -#ifdef TIMING - GPTLstart("PIO:read_darray_nc"); -#endif - ios = file->iosystem; - if(ios == NULL) - return PIO_EBADID; - - vdesc = (file->varlist)+vid; - - if(vdesc == NULL) - return PIO_EBADID; - - ndims = iodesc->ndims; - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - if(fndims==ndims) - vdesc->record=-1; - - if(ios->ioproc){ - io_region *region; - size_t start[fndims]; - size_t count[fndims]; - size_t tmp_start[fndims]; - size_t tmp_count[fndims]; - size_t tmp_bufsize=1; - int regioncnt; - void *bufptr; - int tsize; - - int rrlen=0; - PIO_Offset *startlist[iodesc->maxregions]; - PIO_Offset *countlist[iodesc->maxregions]; - - // buffer is incremented by byte and loffset is in terms of the iodessc->basetype - // so we need to multiply by the size of the basetype - // We can potentially allow for one iodesc to have multiple datatypes by allowing the - // calling program to change the basetype. - region = iodesc->firstregion; - MPI_Type_size(iodesc->basetype, &tsize); - if(fndims>ndims){ - ndims++; - if(vdesc->record<0) - vdesc->record=0; - } - for(regioncnt=0;regioncntmaxregions;regioncnt++){ - // printf("%s %d %d %ld %d %d\n",__FILE__,__LINE__,regioncnt,region,fndims,ndims); - tmp_bufsize=1; - if(region==NULL || iodesc->llen==0){ - for(i=0;iloffset); - - // printf("%s %d %d %d %d\n",__FILE__,__LINE__,iodesc->llen - region->loffset, iodesc->llen, region->loffset); - - if(vdesc->record >= 0 && fndims>1){ - start[0] = vdesc->record; - for(i=1;istart[i-1]; - count[i] = region->count[i-1]; - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); - } - if(count[1]>0) - count[0] = 1; - }else{ - // Non-time dependent array - for(i=0;istart[i]; - count[i] = region->count[i]; - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); - } - } - } - - switch(file->iotype){ -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); - } else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_read_darray\n",(int) iodesc->basetype); - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - { - tmp_bufsize=1; - for(int j=0;j0){ - startlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); - countlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); - - for(int j=0;jmaxregions, j,start[j],count[j],tmp_bufsize); - } - rrlen++; - } - if(regioncnt==iodesc->maxregions-1){ - ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, - countlist, IOBUF, iodesc->llen, iodesc->basetype); - for(i=0;iiotype,__FILE__,__LINE__); - - } - if(region != NULL) - region = region->next; - } // for(regioncnt=0;...) - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); -#ifdef TIMING - GPTLstop("PIO:read_darray_nc"); -#endif - - return ierr; -} - - -/** @brief Read an array of data from a file to the (serial) IO library. - * @ingroup PIO_read_darray - */ -int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) -{ - int ierr=PIO_NOERR; - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ndims, fndims; - MPI_Status status; - int i; - -#ifdef TIMING - GPTLstart("PIO:read_darray_nc_serial"); -#endif - ios = file->iosystem; - if(ios == NULL) - return PIO_EBADID; - - vdesc = (file->varlist)+vid; - - if(vdesc == NULL) - return PIO_EBADID; - - ndims = iodesc->ndims; - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - if(fndims==ndims) - vdesc->record=-1; - - if(ios->ioproc){ - io_region *region; - size_t start[fndims]; - size_t count[fndims]; - size_t tmp_start[fndims*iodesc->maxregions]; - size_t tmp_count[fndims*iodesc->maxregions]; - size_t tmp_bufsize; - int regioncnt; - void *bufptr; - int tsize; - - int rrlen=0; - - // buffer is incremented by byte and loffset is in terms of the iodessc->basetype - // so we need to multiply by the size of the basetype - // We can potentially allow for one iodesc to have multiple datatypes by allowing the - // calling program to change the basetype. - region = iodesc->firstregion; - MPI_Type_size(iodesc->basetype, &tsize); - if(fndims>ndims){ - if(vdesc->record<0) - vdesc->record=0; - } - for(regioncnt=0;regioncntmaxregions;regioncnt++){ - if(region==NULL || iodesc->llen==0){ - for(i=0;irecord >= 0 && fndims>1){ - tmp_start[regioncnt*fndims] = vdesc->record; - for(i=1;istart[i-1]; - tmp_count[i+regioncnt*fndims] = region->count[i-1]; - } - if(tmp_count[1+regioncnt*fndims]>0) - tmp_count[regioncnt*fndims] = 1; - }else{ - // Non-time dependent array - for(i=0;istart[i]; - tmp_count[i+regioncnt*fndims] = region->count[i]; - } - } - /* for(i=0;inext; - } // for(regioncnt=0;...) - - if(ios->io_rank>0){ - MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - if(iodesc->llen > 0){ - MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, 2*ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, 3*ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, 4*ios->num_iotasks+ios->io_rank, ios->io_comm, &status); - } - }else if(ios->io_rank==0){ - int maxregions=0; - size_t loffset, regionsize; - size_t this_start[fndims*iodesc->maxregions]; - size_t this_count[fndims*iodesc->maxregions]; - // for( i=ios->num_iotasks-1; i>=0; i--){ - for(int rtask=1;rtask<=ios->num_iotasks;rtask++){ - if(rtasknum_iotasks){ - MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); - if(tmp_bufsize>0){ - MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, ios->io_comm, &status); - MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, 2*ios->num_iotasks+rtask, ios->io_comm, &status); - MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, 3*ios->num_iotasks+rtask, ios->io_comm, &status); - } - }else{ - maxregions=iodesc->maxregions; - tmp_bufsize=iodesc->llen; - } - loffset = 0; - for(regioncnt=0;regioncntnum_iotasks){ - for(int m=0; mbasetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); - }else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n",(int) iodesc->basetype); - } - - if(ierr != PIO_NOERR){ - for(int i=0;inum_iotasks){ - MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask,4*ios->num_iotasks+rtask, ios->io_comm); - } - } - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); -#ifdef TIMING - GPTLstop("PIO:read_darray_nc_serial"); -#endif - - return ierr; -} - - -/** @brief Read a field from a file to the IO library. - * @ingroup PIO_read_darray - * - */ -int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array) -{ - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - void *iobuf=NULL; - size_t rlen=0; - int ierr, tsize; - MPI_Datatype vtype; - - file = pio_get_file_from_id(ncid); - - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - ios = file->iosystem; - if(ios->iomaster){ - rlen = iodesc->maxiobuflen; - }else{ - rlen = iodesc->llen; - } - - if(iodesc->rearranger > 0){ - if(ios->ioproc && rlen>0){ - MPI_Type_size(iodesc->basetype, &tsize); - iobuf = bget(((size_t) tsize)*rlen); - if(iobuf==NULL){ - piomemerror(*ios,rlen*((size_t) tsize), __FILE__,__LINE__); - } - } - }else{ - iobuf = array; - } - - switch(file->iotype){ - case PIO_IOTYPE_NETCDF: - case PIO_IOTYPE_NETCDF4C: - ierr = pio_read_darray_nc_serial(file, iodesc, vid, iobuf); - break; - case PIO_IOTYPE_PNETCDF: - case PIO_IOTYPE_NETCDF4P: - ierr = pio_read_darray_nc(file, iodesc, vid, iobuf); - break; - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - if(iodesc->rearranger > 0){ - ierr = rearrange_io2comp(*ios, iodesc, iobuf, array); - - if(rlen>0) - brel(iobuf); - } - - return ierr; - -} - -int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) -{ - var_desc_t *vdesc; - int ierr=PIO_NOERR; -#ifdef _PNETCDF - int *status; - PIO_Offset usage = 0; -#ifdef TIMING - GPTLstart("PIO:flush_output_buffer"); -#endif - pioassert(file!=NULL,"file pointer not defined",__FILE__,__LINE__); - - - ierr = ncmpi_inq_buffer_usage(file->fh, &usage); - - if(!force && file->iosystem->io_comm != MPI_COMM_NULL){ - usage += addsize; - - MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, - file->iosystem->io_comm); - } - - if(usage > maxusage){ - maxusage = usage; - } - if(force || usage>=PIO_BUFFER_SIZE_LIMIT){ - int rcnt; - bool prev_dist=false; - int prev_record=-1; - int prev_type=0; - int maxreq; - int reqcnt; - maxreq = 0; - reqcnt=0; - rcnt=0; - for(int i=0; ivarlist+i; - reqcnt+=vdesc->nreqs; - if(vdesc->nreqs>0) maxreq = i; - } - int request[reqcnt]; - int status[reqcnt]; - - for(int i=0; i<=maxreq; i++){ - vdesc = file->varlist+i; -#ifdef MPIO_ONESIDED - /*onesided optimization requires that all of the requests in a wait_all call represent - a contiguous block of data in the file */ - if(rcnt>0 && (prev_record != vdesc->record || - vdesc->nreqs==0)){ - ierr = ncmpi_wait_all(file->fh, rcnt, request,status); - rcnt=0; - } - prev_record = vdesc->record; -#endif - // printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,vdesc->nreqs,vdesc->request); - for(reqcnt=0;reqcntnreqs;reqcnt++){ - request[rcnt++] = max(vdesc->request[reqcnt],NC_REQ_NULL); - } - free(vdesc->request); - vdesc->request=NULL; - vdesc->nreqs=0; - // if(file->iosystem->io_rank < 2) printf("%s %d varid=%d\n",__FILE__,__LINE__,i); -#ifdef FLUSH_EVERY_VAR - ierr = ncmpi_wait_all(file->fh, rcnt, request,status); - rcnt=0; -#endif - } - // if(file->iosystem->io_rank==0){ - // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); - // } - if(rcnt>0){ - /* - if(file->iosystem->io_rank==0){ - printf("%s %d %d ",__FILE__,__LINE__,rcnt); - for(int i=0; ifh, rcnt, request,status); - } - for(int i=0; ivarlist+i; - if(vdesc->iobuf != NULL){ - brel(vdesc->iobuf); - vdesc->iobuf=NULL; - } - if(vdesc->fillbuf != NULL){ - brel(vdesc->fillbuf); - vdesc->fillbuf=NULL; - } - } - - } -#ifdef TIMING - GPTLstop("PIO:flush_output_buffer"); -#endif - -#endif - return ierr; -} - -void cn_buffer_report(iosystem_desc_t ios, bool collective) -{ - - if(CN_bpool != NULL){ - long bget_stats[5]; - long bget_mins[5]; - long bget_maxs[5]; - - bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4); - if(collective){ - MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios.comp_comm); - MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios.comp_comm); - if(ios.compmaster){ - printf("PIO: Currently allocated buffer space %ld %ld\n",bget_mins[0],bget_maxs[0]); - printf("PIO: Currently available buffer space %ld %ld\n",bget_mins[1],bget_maxs[1]); - printf("PIO: Current largest free block %ld %ld\n",bget_mins[2],bget_maxs[2]); - printf("PIO: Number of successful bget calls %ld %ld\n",bget_mins[3],bget_maxs[3]); - printf("PIO: Number of successful brel calls %ld %ld\n",bget_mins[4],bget_maxs[4]); - // print_trace(stdout); - } - }else{ - printf("%d: PIO: Currently allocated buffer space %ld \n",ios.union_rank,bget_stats[0]) ; - printf("%d: PIO: Currently available buffer space %ld \n",ios.union_rank,bget_stats[1]); - printf("%d: PIO: Current largest free block %ld \n",ios.union_rank,bget_stats[2]); - printf("%d: PIO: Number of successful bget calls %ld \n",ios.union_rank,bget_stats[3]); - printf("%d: PIO: Number of successful brel calls %ld \n",ios.union_rank,bget_stats[4]); - } - } -} - -void free_cn_buffer_pool(iosystem_desc_t ios) -{ -#ifndef PIO_USE_MALLOC - if(CN_bpool != NULL){ - cn_buffer_report(ios, true); - bpoolrelease(CN_bpool); - // free(CN_bpool); - CN_bpool=NULL; - } -#endif -} - -void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) -{ - if(wmb->validvars>0){ - PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, wmb->arraylen, wmb->data, wmb->frame, wmb->fillvalue, flushtodisk); - wmb->validvars=0; - brel(wmb->vid); - wmb->vid=NULL; - brel(wmb->data); - wmb->data=NULL; - if(wmb->fillvalue != NULL) - brel(wmb->fillvalue); - if(wmb->frame != NULL) - brel(wmb->frame); - wmb->fillvalue=NULL; - wmb->frame=NULL; - } -} - -void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) -{ - int maxbytesoniotask=INT_MAX; - int maxbytesoncomputetask=INT_MAX; - int maxbytes; - - // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen, iodesc->ndof); - - if(ios.ioproc && iodesc->maxiobuflen>0){ - maxbytesoniotask = PIO_BUFFER_SIZE_LIMIT/ iodesc->maxiobuflen; - } - if(ios.comp_rank>=0 && iodesc->ndof>0){ - maxbytesoncomputetask = PIO_CNBUFFER_LIMIT/iodesc->ndof; - } - maxbytes = min(maxbytesoniotask,maxbytesoncomputetask); - - // printf("%s %d %d %d\n",__FILE__,__LINE__,maxbytesoniotask, maxbytesoncomputetask); - - MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios.union_comm); - iodesc->maxbytes=maxbytes; - // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxbytes,iodesc->maxiobuflen); - -} diff --git a/externals/pio2/src/clib/pio_file.c b/externals/pio2/src/clib/pio_file.c deleted file mode 100644 index 83bf9835a29..00000000000 --- a/externals/pio2/src/clib/pio_file.c +++ /dev/null @@ -1,461 +0,0 @@ -#include -#include -/** - ** @public - ** @ingroup PIO_openfile - ** @brief open an existing file using pio - ** @details Input parameters are read on comp task 0 and ignored elsewhere. - ** @param iosysid : A defined pio system descriptor (input) - ** @param ncidp : A pio file descriptor (output) - ** @param iotype : A pio output format (input) - ** @param filename : The filename to open - ** @param mode : The netcdf mode for the open operation - */ - -int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, - const char filename[], const int mode) -{ - int ierr; - int msg; - int mpierr; - size_t len; - iosystem_desc_t *ios; - file_desc_t *file; - - ierr = PIO_NOERR; - - msg = PIO_MSG_OPEN_FILE; - ios = pio_get_iosystem_from_id(iosysid); - if(ios==NULL){ - printf("bad iosysid %d\n",iosysid); - return PIO_EBADID; - } - - file = (file_desc_t *) malloc(sizeof(*file)); - if(file==NULL){ - return PIO_ENOMEM; - } - file->iotype = *iotype; - file->next = NULL; - file->iosystem = ios; - file->mode = mode; - for(int i=0; ivarlist[i].record = -1; - file->varlist[i].ndims = -1; -#ifdef _PNETCDF - file->varlist[i].request = NULL; - file->varlist[i].nreqs=0; -#endif - file->varlist[i].fillbuf = NULL; - file->varlist[i].iobuf = NULL; - } - - file->buffer.validvars=0; - file->buffer.vid=NULL; - file->buffer.data=NULL; - file->buffer.next=NULL; - file->buffer.frame=NULL; - file->buffer.fillvalue=NULL; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - if(ios->ioproc){ - - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - - case PIO_IOTYPE_NETCDF4P: -#ifdef _MPISERIAL - ierr = nc_open(filename, file->mode, &(file->fh)); -#else - file->mode = file->mode | NC_MPIIO; - ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh)); -#endif - break; - - case PIO_IOTYPE_NETCDF4C: - file->mode = file->mode | NC_NETCDF4; - // *** Note the INTENTIONAL FALLTHROUGH *** -#endif - - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_open(filename, file->mode, &(file->fh)); - } - break; -#endif - -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); - - // This should only be done with a file opened to append - if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){ - if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT); - ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); - } - break; -#endif - - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - break; - } - - // If we failed to open a file due to an incompatible type of NetCDF, try it - // once with just plain old basic NetCDF -#ifdef _NETCDF - if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { - if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); - // reset ierr on all tasks - ierr = PIO_NOERR; - // reset file markers for NETCDF on all tasks - file->iotype = PIO_IOTYPE_NETCDF; - - // open netcdf file serially on main task - if(ios->io_rank==0){ - ierr = nc_open(filename, file->mode, &(file->fh)); } - - } -#endif - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ierr==PIO_NOERR){ - mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); - pio_add_to_file_list(file); - *ncidp = file->fh; - } - if(ios->io_rank==0){ - printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); -// if(file->fh==5) print_trace(stdout); - } - return ierr; -} - -/** - ** @public - ** @ingroup PIO_createfile - ** @brief open a new file using pio - ** @details Input parameters are read on comp task 0 and ignored elsewhere. - ** @param iosysid : A defined pio system descriptor (input) - ** @param ncidp : A pio file descriptor (output) - ** @param iotype : A pio output format (input) - ** @param filename : The filename to open - ** @param mode : The netcdf mode for the open operation - */ - -int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, - const char filename[], const int mode) -{ - int ierr; - int msg; - int mpierr; - - size_t len; - iosystem_desc_t *ios; - file_desc_t *file; - - - ierr = PIO_NOERR; - - ios = pio_get_iosystem_from_id(iosysid); - file = (file_desc_t *) malloc(sizeof(file_desc_t)); - file->next = NULL; - file->iosystem = ios; - file->iotype = *iotype; - - file->buffer.validvars=0; - file->buffer.data=NULL; - file->buffer.next=NULL; - file->buffer.vid=NULL; - file->buffer.ioid=-1; - file->buffer.frame=NULL; - file->buffer.fillvalue=NULL; - - for(int i=0; ivarlist[i].record = -1; - file->varlist[i].ndims = -1; -#ifdef _PNETCDF - file->varlist[i].request = NULL; - file->varlist[i].nreqs=0; -#endif - file->varlist[i].fillbuf = NULL; - file->varlist[i].iobuf = NULL; - } - - msg = PIO_MSG_CREATE_FILE; - file->mode = mode; - - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - // The 64 bit options are not compatable with hdf5 format files - // printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO); - file->mode = file->mode | NC_MPIIO | NC_NETCDF4; - //printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4); - ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh)); - break; - case PIO_IOTYPE_NETCDF4C: - file->mode = file->mode | NC_NETCDF4; -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_create(filename, file->mode, &(file->fh)); - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); - if(ierr == PIO_NOERR){ - if(ios->io_rank==0){ - printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT); - } - int oldfill; - ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); - // ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill); - } - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ierr == PIO_NOERR){ - mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); - file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set - pio_add_to_file_list(file); - *ncidp = file->fh; - } - if(ios->io_rank==0){ - printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); -// if(file->fh==5) print_trace(stdout); - } - return ierr; -} - -/** - ** @ingroup PIO_closefile - ** @brief close a file previously opened with PIO - ** @param ncid: the file pointer - */ -int PIOc_closefile(int ncid) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = 0; - if((file->mode & PIO_WRITE)){ - PIOc_sync(ncid); - } - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_close(file->fh); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_close(file->fh); - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - if((file->mode & PIO_WRITE)){ - ierr = ncmpi_buffer_detach(file->fh); - } - ierr = ncmpi_close(file->fh); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - if(ios->io_rank==0){ - printf("Close file %d \n",file->fh); -// if(file->fh==5) print_trace(stdout); - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - int iret = pio_delete_file_from_list(ncid); - - - return ierr; -} - -/** - ** @ingroup PIO_deletefile - ** @brief Delete a file - ** @param iosysid : a pio system handle - ** @param filename : a filename - */ -int PIOc_deletefile(const int iosysid, const char filename[]) -{ - int ierr; - int msg; - int mpierr; - int chkerr; - iosystem_desc_t *ios; - - ierr = PIO_NOERR; - ios = pio_get_iosystem_from_id(iosysid); - - if(ios == NULL) - return PIO_EBADID; - - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - // mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm); - } - // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. - if(ios->ioproc){ - MPI_Barrier(ios->io_comm); -#ifdef _NETCDF - if(ios->io_rank==0) - ierr = nc_delete(filename); -#else -#ifdef _PNETCDF - ierr = ncmpi_delete(filename, ios->info); -#endif -#endif - MPI_Barrier(ios->io_comm); - } - // Special case - always broadcast the return from the - MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); - - - - return ierr; -} - -/// -/// PIO interface to nc_sync -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -/** -* @name PIOc_sync -*/ -int PIOc_sync (int ncid) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - wmulti_buffer *wmb, *twmb; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_SYNC; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if((file->mode & PIO_WRITE)){ - // cn_buffer_report( *ios, true); - wmb = &(file->buffer); - while(wmb != NULL){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,wmb->ioid, wmb->validvars); - if(wmb->validvars>0){ - flush_buffer(ncid, wmb, true); - } - twmb = wmb; - wmb = wmb->next; - if(twmb == &(file->buffer)){ - twmb->ioid=-1; - twmb->next=NULL; - }else{ - brel(twmb); - } - } - flush_output_buffer(file, true, 0); - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_sync(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_sync(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_sync(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - } - return ierr; -} - diff --git a/externals/pio2/src/clib/pio_get_nc.c b/externals/pio2/src/clib/pio_get_nc.c deleted file mode 100644 index 0da690aaf21..00000000000 --- a/externals/pio2/src/clib/pio_get_nc.c +++ /dev/null @@ -1,4988 +0,0 @@ -#include -#include - -int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SCHAR; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_double (int ncid, int varid, double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_double(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_double(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_double(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_double_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_double(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_int (int ncid, int varid, int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_int(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_int(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_int(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_int_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_ushort (int ncid, int varid, unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_ushort(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_ushort(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_ushort(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_ushort_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_text(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_int(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_FLOAT; - ibuftype = MPI_FLOAT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SHORT; - ibuftype = MPI_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_int_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_text (int ncid, int varid, char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_text(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_text(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_text(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_text_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_float (int ncid, int varid, float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_float(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_float(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_float(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_float_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONGLONG; - ibuftype = MPI_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_long (int ncid, int varid, long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_long(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_long(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_long(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_long_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_DOUBLE; - ibuftype = MPI_DOUBLE; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_uint(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_longlong (int ncid, int varid, long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_longlong(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_longlong(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_longlong(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_longlong_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_short(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_long(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_INT; - ibuftype = MPI_INT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_uchar(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_uchar(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_uchar(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_uchar_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_uint(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_uint(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_uint(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_uint_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_schar(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UINT; - ibuftype = MPI_UNSIGNED; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_float(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_TEXT; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_ulonglong(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_ulonglong(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_ulonglong(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_ulonglong_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_short (int ncid, int varid, short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_short(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_short(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_short(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_short_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONG; - ibuftype = MPI_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_uchar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_all(file->fh, varid, start, count, stride, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_schar (int ncid, int varid, signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_schar(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_schar(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_schar(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_schar_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - diff --git a/externals/pio2/src/clib/pio_internal.h b/externals/pio2/src/clib/pio_internal.h deleted file mode 100644 index f7a3c9ddf22..00000000000 --- a/externals/pio2/src/clib/pio_internal.h +++ /dev/null @@ -1,371 +0,0 @@ -#ifndef __PIO_INTERNAL__ -#define __PIO_INTERNAL__ -#include -// It seems that some versions of openmpi fail to define MPI_OFFSET -#ifdef OMPI_OFFSET_DATATYPE -#ifndef MPI_OFFSET -#define MPI_OFFSET OMPI_OFFSET_DATATYPE -#endif -#endif -#ifndef MPI_Offset -#define MPI_Offset long long -#endif -#include -#include -#include -#ifdef TIMING -#include -#endif - - -#define max(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a > _b ? _a : _b; }) - -#define min(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a < _b ? _a : _b; }) - - -#define MAX_GATHER_BLOCK_SIZE 0 -#define PIO_REQUEST_ALLOC_CHUNK 16 - -#if defined(__cplusplus) -extern "C" { -#endif - -extern PIO_Offset PIO_BUFFER_SIZE_LIMIT; -extern bool PIO_Save_Decomps; - - -/** - ** @brief Used to sort map points in the subset rearranger -*/ -typedef struct mapsort -{ - int rfrom; - PIO_Offset soffset; - PIO_Offset iomap; -} mapsort; - -/** - * @brief swapm defaults. - * -*/ -typedef struct pio_swapm_defaults -{ - int nreqs; - bool handshake; - bool isend; -} pio_swapm_defaults; - - - void pio_get_env(void); - int pio_add_to_iodesc_list(io_desc_t *iodesc); - io_desc_t *pio_get_iodesc_from_id(int ioid); - int pio_delete_iodesc_from_list(int ioid); - - file_desc_t *pio_get_file_from_id(int ncid); - int pio_delete_file_from_list(int ncid); - void pio_add_to_file_list(file_desc_t *file); - void pio_push_request(file_desc_t *file, int request); - - iosystem_desc_t *pio_get_iosystem_from_id(int iosysid); - int pio_add_to_iosystem_list(iosystem_desc_t *ios); - - int check_netcdf(file_desc_t *file,const int status, const char *fname, const int line); - int iotype_error(const int iotype, const char *fname, const int line); - void piodie(const char *msg,const char *fname, const int line); - void pioassert(bool exp, const char *msg,const char *fname, const int line); - int CalcStartandCount(const int basetype, const int ndims, const int *gdims, const int num_io_procs, - const int myiorank, PIO_Offset *start, PIO_Offset *kount); - void CheckMPIReturn(const int ierr,const char file[],const int line); - int pio_fc_gather( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnt, const MPI_Datatype recvtype, const int root, - MPI_Comm comm, const int flow_cntl); - int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnts[], const int recvdispl[], const MPI_Datatype recvtype, const int root, - MPI_Comm comm, const int flow_cntl); - - int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnts[], const int rdispls[], const MPI_Datatype recvtype, const int root, - MPI_Comm comm, const int flow_cntl); - - int pio_swapm(void *sndbuf, int sndlths[], int sdispls[], MPI_Datatype stypes[], - void *rcvbuf, int rcvlths[], int rdispls[], MPI_Datatype rtypes[], - MPI_Comm comm, const bool handshake, bool isend, const int max_requests); - - long long lgcd_array(int nain, long long*ain); - - void PIO_Offset_size(MPI_Datatype *dtype, int *tsize); - PIO_Offset GCDblocksize(const int arrlen, const PIO_Offset arr_in[]); - - int subset_rearrange_create(const iosystem_desc_t ios,const int maplen, PIO_Offset compmap[], const int gsize[], - const int ndim, io_desc_t *iodesc); - - - int box_rearrange_create(const iosystem_desc_t ios,const int maplen, const PIO_Offset compmap[], const int gsize[], - const int ndim, io_desc_t *iodesc); - - - int rearrange_io2comp(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, - void *rbuf); - int rearrange_comp2io(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, - void *rbuf, const int nvars); - int calcdisplace(const int bsize, const int numblocks,const PIO_Offset map[],int displace[]); - io_desc_t *malloc_iodesc(const int piotype, const int ndims); - void performance_tune_rearranger(iosystem_desc_t ios, io_desc_t *iodesc); - - int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize); - void compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc); - io_region *alloc_region(const int ndims); - int pio_delete_iosystem_from_list(int piosysid); - int gcd(int a, int b); - long long lgcd (long long a,long long b ); - int gcd_array(int nain, int *ain); - void free_region_list(io_region *top); - void gindex_to_coord(const int ndims, const PIO_Offset gindex, const PIO_Offset gstride[], PIO_Offset *gcoord); - PIO_Offset coord_to_lindex(const int ndims, const PIO_Offset lcoord[], const PIO_Offset count[]); - - int ceil2(const int i); - int pair(const int np, const int p, const int k); - int define_iodesc_datatypes(const iosystem_desc_t ios, io_desc_t *iodesc); - - int create_mpi_datatypes(const MPI_Datatype basetype,const int msgcnt,const PIO_Offset dlen, - const PIO_Offset mindex[],const int mcount[],int *mfrom, MPI_Datatype mtype[]); - int compare_offsets(const void *a,const void *b) ; - - int subset_rearrange_create(const iosystem_desc_t ios, int maplen, PIO_Offset compmap[], - const int gsize[], const int ndims, io_desc_t *iodesc); - void print_trace (FILE *fp); - void cn_buffer_report(iosystem_desc_t ios, bool collective); - void compute_buffer_init(iosystem_desc_t ios); - void free_cn_buffer_pool(iosystem_desc_t ios); -void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); - void piomemerror(iosystem_desc_t ios, size_t req, char *fname, const int line); - void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc); - -#ifdef BGQ - void identity(MPI_Comm comm, int *iotask); - void determineiotasks(const MPI_Comm comm, int *numiotasks,int *base, int *stride, int *rearr, - bool *iamIOtask); - -#endif - -#if defined(__cplusplus) -} -#endif - -enum PIO_MSG{ - PIO_MSG_OPEN_FILE, - PIO_MSG_CREATE_FILE, - PIO_MSG_INQ_ATT, - PIO_MSG_INQ_FORMAT, - PIO_MSG_INQ_VARID, - PIO_MSG_INQ_VARNATTS, - PIO_MSG_DEF_VAR, - PIO_MSG_INQ_VAR, - PIO_MSG_INQ_VARNAME, - PIO_MSG_PUT_ATT_DOUBLE, - PIO_MSG_PUT_ATT_INT, - PIO_MSG_RENAME_ATT, - PIO_MSG_DEL_ATT, - PIO_MSG_INQ_NATTS, - PIO_MSG_INQ, - PIO_MSG_GET_ATT_TEXT, - PIO_MSG_GET_ATT_SHORT, - PIO_MSG_PUT_ATT_LONG, - PIO_MSG_REDEF, - PIO_MSG_SET_FILL, - PIO_MSG_ENDDEF, - PIO_MSG_RENAME_VAR, - PIO_MSG_PUT_ATT_SHORT, - PIO_MSG_PUT_ATT_TEXT, - PIO_MSG_INQ_ATTNAME, - PIO_MSG_GET_ATT_ULONGLONG, - PIO_MSG_GET_ATT_USHORT, - PIO_MSG_PUT_ATT_ULONGLONG, - PIO_MSG_INQ_DIMLEN, - PIO_MSG_GET_ATT_UINT, - PIO_MSG_GET_ATT_LONGLONG, - PIO_MSG_PUT_ATT_SCHAR, - PIO_MSG_PUT_ATT_FLOAT, - PIO_MSG_INQ_NVARS, - PIO_MSG_RENAME_DIM, - PIO_MSG_INQ_VARNDIMS, - PIO_MSG_GET_ATT_LONG, - PIO_MSG_INQ_DIM, - PIO_MSG_INQ_DIMID, - PIO_MSG_INQ_UNLIMDIM, - PIO_MSG_INQ_VARDIMID, - PIO_MSG_INQ_ATTLEN, - PIO_MSG_INQ_DIMNAME, - PIO_MSG_PUT_ATT_USHORT, - PIO_MSG_GET_ATT_FLOAT, - PIO_MSG_SYNC, - PIO_MSG_PUT_ATT_LONGLONG, - PIO_MSG_PUT_ATT_UINT, - PIO_MSG_GET_ATT_SCHAR, - PIO_MSG_INQ_ATTID, - PIO_MSG_DEF_DIM, - PIO_MSG_INQ_NDIMS, - PIO_MSG_INQ_VARTYPE, - PIO_MSG_GET_ATT_INT, - PIO_MSG_GET_ATT_DOUBLE, - PIO_MSG_INQ_ATTTYPE, - PIO_MSG_PUT_ATT_UCHAR, - PIO_MSG_GET_ATT_UCHAR, - PIO_MSG_PUT_VARS_UCHAR, - PIO_MSG_GET_VAR1_SCHAR, - PIO_MSG_GET_VARS_ULONGLONG, - PIO_MSG_GET_VARM_UCHAR, - PIO_MSG_GET_VARM_SCHAR, - PIO_MSG_GET_VARS_SHORT, - PIO_MSG_GET_VAR_DOUBLE, - PIO_MSG_GET_VARA_DOUBLE, - PIO_MSG_GET_VAR_INT, - PIO_MSG_GET_VAR_USHORT, - PIO_MSG_PUT_VARS_USHORT, - PIO_MSG_GET_VARA_TEXT, - PIO_MSG_PUT_VARS_ULONGLONG, - PIO_MSG_GET_VARA_INT, - PIO_MSG_PUT_VARM, - PIO_MSG_GET_VAR1_FLOAT, - PIO_MSG_GET_VAR1_SHORT, - PIO_MSG_GET_VARS_INT, - PIO_MSG_PUT_VARS_UINT, - PIO_MSG_GET_VAR_TEXT, - PIO_MSG_GET_VARM_DOUBLE, - PIO_MSG_PUT_VARM_UCHAR, - PIO_MSG_PUT_VAR_USHORT, - PIO_MSG_GET_VARS_SCHAR, - PIO_MSG_GET_VARA_USHORT, - PIO_MSG_PUT_VAR1_LONGLONG, - PIO_MSG_PUT_VARA_UCHAR, - PIO_MSG_PUT_VARM_SHORT, - PIO_MSG_PUT_VAR1_LONG, - PIO_MSG_PUT_VARS_LONG, - PIO_MSG_GET_VAR1_USHORT, - PIO_MSG_PUT_VAR_SHORT, - PIO_MSG_PUT_VARA_INT, - PIO_MSG_GET_VAR_FLOAT, - PIO_MSG_PUT_VAR1_USHORT, - PIO_MSG_PUT_VARA_TEXT, - PIO_MSG_PUT_VARM_TEXT, - PIO_MSG_GET_VARS_UCHAR, - PIO_MSG_GET_VAR, - PIO_MSG_PUT_VARM_USHORT, - PIO_MSG_GET_VAR1_LONGLONG, - PIO_MSG_GET_VARS_USHORT, - PIO_MSG_GET_VAR_LONG, - PIO_MSG_GET_VAR1_DOUBLE, - PIO_MSG_PUT_VAR_ULONGLONG, - PIO_MSG_PUT_VAR_INT, - PIO_MSG_GET_VARA_UINT, - PIO_MSG_PUT_VAR_LONGLONG, - PIO_MSG_GET_VARS_LONGLONG, - PIO_MSG_PUT_VAR_SCHAR, - PIO_MSG_PUT_VAR_UINT, - PIO_MSG_PUT_VAR, - PIO_MSG_PUT_VARA_USHORT, - PIO_MSG_GET_VAR_LONGLONG, - PIO_MSG_GET_VARA_SHORT, - PIO_MSG_PUT_VARS_SHORT, - PIO_MSG_PUT_VARA_UINT, - PIO_MSG_PUT_VARA_SCHAR, - PIO_MSG_PUT_VARM_ULONGLONG, - PIO_MSG_PUT_VAR1_UCHAR, - PIO_MSG_PUT_VARM_INT, - PIO_MSG_PUT_VARS_SCHAR, - PIO_MSG_GET_VARA_LONG, - PIO_MSG_PUT_VAR1, - PIO_MSG_GET_VAR1_INT, - PIO_MSG_GET_VAR1_ULONGLONG, - PIO_MSG_GET_VAR_UCHAR, - PIO_MSG_PUT_VARA_FLOAT, - PIO_MSG_GET_VARA_UCHAR, - PIO_MSG_GET_VARS_FLOAT, - PIO_MSG_PUT_VAR1_FLOAT, - PIO_MSG_PUT_VARM_FLOAT, - PIO_MSG_PUT_VAR1_TEXT, - PIO_MSG_PUT_VARS_TEXT, - PIO_MSG_PUT_VARM_LONG, - PIO_MSG_GET_VARS_LONG, - PIO_MSG_PUT_VARS_DOUBLE, - PIO_MSG_GET_VAR1, - PIO_MSG_GET_VAR_UINT, - PIO_MSG_PUT_VARA_LONGLONG, - PIO_MSG_GET_VARA, - PIO_MSG_PUT_VAR_DOUBLE, - PIO_MSG_GET_VARA_SCHAR, - PIO_MSG_PUT_VAR_FLOAT, - PIO_MSG_GET_VAR1_UINT, - PIO_MSG_GET_VARS_UINT, - PIO_MSG_PUT_VAR1_ULONGLONG, - PIO_MSG_PUT_VARM_UINT, - PIO_MSG_PUT_VAR1_UINT, - PIO_MSG_PUT_VAR1_INT, - PIO_MSG_GET_VARA_FLOAT, - PIO_MSG_GET_VARM_TEXT, - PIO_MSG_PUT_VARS_FLOAT, - PIO_MSG_GET_VAR1_TEXT, - PIO_MSG_PUT_VARA_SHORT, - PIO_MSG_PUT_VAR1_SCHAR, - PIO_MSG_PUT_VARA_ULONGLONG, - PIO_MSG_PUT_VARM_DOUBLE, - PIO_MSG_GET_VARM_INT, - PIO_MSG_PUT_VARA, - PIO_MSG_PUT_VARA_LONG, - PIO_MSG_GET_VARM_UINT, - PIO_MSG_GET_VARM, - PIO_MSG_PUT_VAR1_DOUBLE, - PIO_MSG_GET_VARS_DOUBLE, - PIO_MSG_GET_VARA_LONGLONG, - PIO_MSG_GET_VAR_ULONGLONG, - PIO_MSG_PUT_VARM_SCHAR, - PIO_MSG_GET_VARA_ULONGLONG, - PIO_MSG_GET_VAR_SHORT, - PIO_MSG_GET_VARM_FLOAT, - PIO_MSG_PUT_VAR_TEXT, - PIO_MSG_PUT_VARS_INT, - PIO_MSG_GET_VAR1_LONG, - PIO_MSG_GET_VARM_LONG, - PIO_MSG_GET_VARM_USHORT, - PIO_MSG_PUT_VAR1_SHORT, - PIO_MSG_PUT_VARS_LONGLONG, - PIO_MSG_GET_VARM_LONGLONG, - PIO_MSG_GET_VARS_TEXT, - PIO_MSG_PUT_VARA_DOUBLE, - PIO_MSG_PUT_VARS, - PIO_MSG_PUT_VAR_UCHAR, - PIO_MSG_GET_VAR1_UCHAR, - PIO_MSG_PUT_VAR_LONG, - PIO_MSG_GET_VARS, - PIO_MSG_GET_VARM_SHORT, - PIO_MSG_GET_VARM_ULONGLONG, - PIO_MSG_PUT_VARM_LONGLONG, - PIO_MSG_GET_VAR_SCHAR, - PIO_MSG_GET_ATT_UBYTE, - PIO_MSG_PUT_ATT_STRING, - PIO_MSG_GET_ATT_STRING, - PIO_MSG_PUT_ATT_UBYTE, - PIO_MSG_INQ_VAR_FILL, - PIO_MSG_DEF_VAR_FILL, - PIO_MSG_DEF_VAR_DEFLATE, - PIO_MSG_INQ_VAR_DEFLATE, - PIO_MSG_INQ_VAR_SZIP, - PIO_MSG_DEF_VAR_FLETCHER32, - PIO_MSG_INQ_VAR_FLETCHER32, - PIO_MSG_DEF_VAR_CHUNKING, - PIO_MSG_INQ_VAR_CHUNKING, - PIO_MSG_DEF_VAR_ENDIAN, - PIO_MSG_INQ_VAR_ENDIAN, - PIO_MSG_SET_CHUNK_CACHE, - PIO_MSG_GET_CHUNK_CACHE, - PIO_MSG_SET_VAR_CHUNK_CACHE, - PIO_MSG_GET_VAR_CHUNK_CACHE -}; - -#endif diff --git a/externals/pio2/src/clib/pio_lists.c b/externals/pio2/src/clib/pio_lists.c deleted file mode 100644 index 511918b52b5..00000000000 --- a/externals/pio2/src/clib/pio_lists.c +++ /dev/null @@ -1,221 +0,0 @@ -#include -#include -#include -#include - -static io_desc_t *pio_iodesc_list=NULL; -static io_desc_t *current_iodesc=NULL; -static iosystem_desc_t *pio_iosystem_list=NULL; -static file_desc_t *pio_file_list = NULL; -static file_desc_t *current_file=NULL; - -void pio_add_to_file_list(file_desc_t *file) -{ - file_desc_t *cfile; - int cnt=-1; - // on iotasks the fh returned from netcdf should be unique, on non-iotasks we - // need to generate a unique fh, we do this with cnt, its a negative index - - file->next = NULL; - cfile = pio_file_list; - current_file = file; - if(cfile==NULL){ - pio_file_list = file; - }else{ - cnt = min(cnt,cfile->fh-1); - while(cfile->next != NULL) - { - cfile=cfile->next; - cnt = min(cnt,cfile->fh-1); - } - cfile->next = file; - } - if(! file->iosystem->ioproc || ((file->iotype != PIO_IOTYPE_PNETCDF && - file->iotype != PIO_IOTYPE_NETCDF4P) && - file->iosystem->io_rank>0)) - file->fh = cnt; - - cfile = pio_file_list; - -} - - - -file_desc_t *pio_get_file_from_id(int ncid) -{ - file_desc_t *cfile; - - cfile = NULL; - - if(current_file != NULL && current_file->fh == ncid) - cfile=current_file; - for(cfile=pio_file_list; cfile != NULL; cfile=cfile->next){ - if(cfile->fh == ncid){ - current_file = cfile; - break; - } - } - - if(cfile==NULL){ - printf("ERROR: Operation on undefined file %s %d\n",__FILE__,__LINE__); - } - return cfile; -} - -int pio_delete_file_from_list(int ncid) -{ - - file_desc_t *cfile, *pfile; - - pfile = NULL; - - for(cfile=pio_file_list; cfile != NULL; cfile=cfile->next){ - if(cfile->fh == ncid){ - if(pfile == NULL){ - pio_file_list = cfile->next; - }else{ - pfile->next = cfile->next; - } - if(current_file==cfile) - current_file=pfile; - free(cfile); - return PIO_NOERR; - } - pfile = cfile; - } - return PIO_EBADID; -} - -int pio_delete_iosystem_from_list(int piosysid) -{ - - iosystem_desc_t *ciosystem, *piosystem; - - piosystem = NULL; - - // printf(" %d iosystem_top = %ld \n",__LINE__,pio_iosystem_list); - - for(ciosystem=pio_iosystem_list; ciosystem != NULL; ciosystem=ciosystem->next){ - if(ciosystem->iosysid == piosysid){ - if(piosystem == NULL){ - pio_iosystem_list = ciosystem->next; - }else{ - piosystem->next = ciosystem->next; - } - free(ciosystem); - return PIO_NOERR; - } - piosystem = ciosystem; - } - return PIO_EBADID; -} - -int pio_add_to_iosystem_list(iosystem_desc_t *ios) -{ - iosystem_desc_t *cios; - int i=1; - - //assert(ios != NULL); - ios->next = NULL; - cios = pio_iosystem_list; - if(cios==NULL) - pio_iosystem_list = ios; - else{ - i++; - while(cios->next != NULL){ - cios = cios->next; - i++; - } - cios->next = ios; - } - ios->iosysid = i << 16; - // ios->iosysid = i ; - // printf(" ios = %ld %d %ld\n",ios, ios->iosysid,ios->next); - return ios->iosysid; -} - -iosystem_desc_t *pio_get_iosystem_from_id(int iosysid) -{ - iosystem_desc_t *ciosystem; - - ciosystem = pio_iosystem_list; - - while(ciosystem != NULL){ - // fprintf(stderr, "%d ciosystem %ld %ld %d\n",__LINE__,pio_iosystem_list, ciosystem,iosysid); - if(ciosystem->iosysid == iosysid){ - return ciosystem; - } - ciosystem = ciosystem->next; - } - return NULL; - -} - -int pio_add_to_iodesc_list(io_desc_t *iodesc) -{ - io_desc_t *ciodesc; - int imax=512; - - iodesc->next = NULL; - if(pio_iodesc_list == NULL) - pio_iodesc_list = iodesc; - else{ - imax++; - for(ciodesc = pio_iodesc_list; ciodesc->next != NULL; ciodesc=ciodesc->next, imax=ciodesc->ioid+1); - ciodesc->next = iodesc; - } - iodesc->ioid = imax; - current_iodesc = iodesc; - // printf("In add to list %d\n",iodesc->ioid); - return iodesc->ioid; -} - - -io_desc_t *pio_get_iodesc_from_id(int ioid) -{ - io_desc_t *ciodesc; - - ciodesc = NULL; - - if(current_iodesc != NULL && current_iodesc->ioid == abs(ioid)) - ciodesc=current_iodesc; - for(ciodesc=pio_iodesc_list; ciodesc != NULL; ciodesc=ciodesc->next){ - if(ciodesc->ioid == abs(ioid)){ - current_iodesc = ciodesc; - break; - } - } - /* - if(ciodesc==NULL){ - for(ciodesc=pio_iodesc_list; ciodesc != NULL; ciodesc=ciodesc->next){ - printf("%s %d %d %d\n",__FILE__,__LINE__,ioid,ciodesc->ioid); - } - } - */ - - return ciodesc; -} - -int pio_delete_iodesc_from_list(int ioid) -{ - - io_desc_t *ciodesc, *piodesc; - - piodesc = NULL; - // if(abs(ioid)==518) printf("In delete from list %d\n", ioid); - for(ciodesc=pio_iodesc_list; ciodesc != NULL; ciodesc=ciodesc->next){ - if(ciodesc->ioid == ioid){ - if(piodesc == NULL){ - pio_iodesc_list = ciodesc->next; - }else{ - piodesc->next = ciodesc->next; - } - if(current_iodesc==ciodesc) - current_iodesc=pio_iodesc_list; - brel(ciodesc); - return PIO_NOERR; - } - piodesc = ciodesc; - } - return PIO_EBADID; -} diff --git a/externals/pio2/src/clib/pio_nc.c b/externals/pio2/src/clib/pio_nc.c deleted file mode 100644 index f511475a4f6..00000000000 --- a/externals/pio2/src/clib/pio_nc.c +++ /dev/null @@ -1,4404 +0,0 @@ -/** -* @file -* @author Jim Edwards (jedwards@ucar.edu) -* @date Feburary 2014 -* @brief PIO interfaces to [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) support functions -* @details -* This file provides an interface to the [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) support functions. -* Each subroutine calls the underlying netcdf or pnetcdf or netcdf4 functions from -* the appropriate subset of mpi tasks (io_comm). Each routine must be called -* collectively from union_comm. -* -*/ -#include -#include - -/** - * @ingroup PIOc_inq - * The PIO-C interface for the NetCDF function nc_inq. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ndimsp != NULL) - mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(nvarsp != NULL) - mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(ngattsp != NULL) - mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(unlimdimidp != NULL) - mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_dimname - * The PIO-C interface for the NetCDF function nc_inq_dimname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_dimname (int ncid, int dimid, char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_DIMNAME; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimname(file->fh, dimid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_dimname(file->fh, dimid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimname(file->fh, dimid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_short - * The PIO-C interface for the NetCDF function nc_put_att_short. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_rename_dim - * The PIO-C interface for the NetCDF function nc_rename_dim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_rename_dim (int ncid, int dimid, const char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_RENAME_DIM; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_dim(file->fh, dimid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_dim(file->fh, dimid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_dim(file->fh, dimid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_double - * The PIO-C interface for the NetCDF function nc_get_att_double. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_double(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_double(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_set_fill - * The PIO-C interface for the NetCDF function nc_set_fill. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_set_fill (int ncid, int fillmode, int *old_modep) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_SET_FILL; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_set_fill(file->fh, fillmode, old_modep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_set_fill(file->fh, fillmode, old_modep);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_def_var - * The PIO-C interface for the NetCDF function nc_def_var. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param varidp a pointer that will get the variable id - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - break; - case PIO_IOTYPE_NETCDF4C: - if(ios->io_rank==0){ - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); - if(ierr == PIO_NOERR){ - ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); - } - } - break; -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_double - * The PIO-C interface for the NetCDF function nc_put_att_double. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_dim - * The PIO-C interface for the NetCDF function nc_inq_dim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_DIM; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_uchar - * The PIO-C interface for the NetCDF function nc_get_att_uchar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_var_fill - * The PIO-C interface for the NetCDF function nc_inq_var_fill. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_FILL; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_attid - * The PIO-C interface for the NetCDF function nc_inq_attid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param idp a pointer that will get the id of the variable or attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTID; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attid(file->fh, varid, name, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attid(file->fh, varid, name, idp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_vartype - * The PIO-C interface for the NetCDF function nc_inq_vartype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARTYPE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_vartype(file->fh, varid, xtypep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_vartype(file->fh, varid, xtypep);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_schar - * The PIO-C interface for the NetCDF function nc_put_att_schar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_vardimid - * The PIO-C interface for the NetCDF function nc_inq_vardimid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARDIMID; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr==PIO_NOERR){ - int ndims; - PIOc_inq_varndims(file->fh, varid, &ndims); - mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_ushort - * The PIO-C interface for the NetCDF function nc_get_att_ushort. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_varid - * The PIO-C interface for the NetCDF function nc_inq_varid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param varidp a pointer that will get the variable id - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varid (int ncid, const char *name, int *varidp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARID; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varid(file->fh, name, varidp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varid(file->fh, name, varidp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varid(file->fh, name, varidp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(varidp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_attlen - * The PIO-C interface for the NetCDF function nc_inq_attlen. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTLEN; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_atttype - * The PIO-C interface for the NetCDF function nc_inq_atttype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTTYPE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_rename_var - * The PIO-C interface for the NetCDF function nc_rename_var. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_rename_var (int ncid, int varid, const char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_RENAME_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_var(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_var(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_var(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_natts - * The PIO-C interface for the NetCDF function nc_inq_natts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_natts (int ncid, int *ngattsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_NATTS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_natts(file->fh, ngattsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_natts(file->fh, ngattsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_natts(file->fh, ngattsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_ulonglong - * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_var - * The PIO-C interface for the NetCDF function nc_inq_var. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param nattsp a pointer that will get the number of attributes - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(ndimsp != NULL){ mpierr = MPI_Bcast(ndimsp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp);} - if(nattsp != NULL) mpierr = MPI_Bcast(nattsp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(dimidsp != NULL) {int ndims; - PIOc_inq_varndims(file->fh, varid, &ndims); - mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_rename_att - * The PIO-C interface for the NetCDF function nc_rename_att. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_RENAME_ATT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_att(file->fh, varid, name, newname);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_att(file->fh, varid, name, newname);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_att(file->fh, varid, name, newname);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_ushort - * The PIO-C interface for the NetCDF function nc_put_att_ushort. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_dimid - * The PIO-C interface for the NetCDF function nc_inq_dimid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param idp a pointer that will get the id of the variable or attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_dimid (int ncid, const char *name, int *idp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_DIMID; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimid(file->fh, name, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_dimid(file->fh, name, idp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimid(file->fh, name, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_text - * The PIO-C interface for the NetCDF function nc_put_att_text. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_uint - * The PIO-C interface for the NetCDF function nc_get_att_uint. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uint(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_uint(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_format - * The PIO-C interface for the NetCDF function nc_inq_format. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param formatp a pointer that will get the file format - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_format (int ncid, int *formatp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_FORMAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_format(file->fh, formatp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_format(file->fh, formatp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_format(file->fh, formatp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_long - * The PIO-C interface for the NetCDF function nc_get_att_long. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_long(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_long(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_attname - * The PIO-C interface for the NetCDF function nc_inq_attname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param attnum the attribute ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTNAME; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attname(file->fh, varid, attnum, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attname(file->fh, varid, attnum, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_att - * The PIO-C interface for the NetCDF function nc_inq_att. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_long - * The PIO-C interface for the NetCDF function nc_put_att_long. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_unlimdim - * The PIO-C interface for the NetCDF function nc_inq_unlimdim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_UNLIMDIM; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_float - * The PIO-C interface for the NetCDF function nc_get_att_float. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_float(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_float(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_ndims - * The PIO-C interface for the NetCDF function nc_inq_ndims. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_ndims (int ncid, int *ndimsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_NDIMS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_ndims(file->fh, ndimsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_ndims(file->fh, ndimsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_ndims(file->fh, ndimsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_int - * The PIO-C interface for the NetCDF function nc_put_att_int. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_nvars - * The PIO-C interface for the NetCDF function nc_inq_nvars. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_nvars (int ncid, int *nvarsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_NVARS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_nvars(file->fh, nvarsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_nvars(file->fh, nvarsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_nvars(file->fh, nvarsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_enddef - * The PIO-C interface for the NetCDF function nc_enddef. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_enddef (int ncid) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_ENDDEF; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_enddef(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_enddef(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_enddef(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_uchar - * The PIO-C interface for the NetCDF function nc_put_att_uchar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_longlong - * The PIO-C interface for the NetCDF function nc_put_att_longlong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_varnatts - * The PIO-C interface for the NetCDF function nc_inq_varnatts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param nattsp a pointer that will get the number of attributes - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARNATTS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varnatts(file->fh, varid, nattsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varnatts(file->fh, varid, nattsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(nattsp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_ubyte - * The PIO-C interface for the NetCDF function nc_get_att_ubyte. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UBYTE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_text - * The PIO-C interface for the NetCDF function nc_get_att_text. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_text(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_text(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_text(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_del_att - * The PIO-C interface for the NetCDF function nc_del_att. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_del_att (int ncid, int varid, const char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEL_ATT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_del_att(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_del_att(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_del_att(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_dimlen - * The PIO-C interface for the NetCDF function nc_inq_dimlen. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_DIMLEN; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_schar - * The PIO-C interface for the NetCDF function nc_get_att_schar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_schar(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_schar(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_ulonglong - * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_varndims - * The PIO-C interface for the NetCDF function nc_inq_varndims. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARNDIMS; - if(file->varlist[varid].ndims > 0){ - (*ndimsp) = file->varlist[varid].ndims; - return PIO_NOERR; - } - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varndims(file->fh, varid, ndimsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varndims(file->fh, varid, ndimsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_varname - * The PIO-C interface for the NetCDF function nc_inq_varname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varname (int ncid, int varid, char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARNAME; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varname(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varname(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varname(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_def_dim - * The PIO-C interface for the NetCDF function nc_def_dim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param idp a pointer that will get the id of the variable or attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_DIM; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_dim(file->fh, name, len, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_uint - * The PIO-C interface for the NetCDF function nc_put_att_uint. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_short - * The PIO-C interface for the NetCDF function nc_get_att_short. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_short(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_short(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_short(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_redef - * The PIO-C interface for the NetCDF function nc_redef. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_redef (int ncid) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_REDEF; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_redef(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_redef(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_redef(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_ubyte - * The PIO-C interface for the NetCDF function nc_put_att_ubyte. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UBYTE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ubyte(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_int - * The PIO-C interface for the NetCDF function nc_get_att_int. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_int(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_int(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_int(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_longlong - * The PIO-C interface for the NetCDF function nc_get_att_longlong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_longlong(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_longlong(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_float - * The PIO-C interface for the NetCDF function nc_put_att_float. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - diff --git a/externals/pio2/src/clib/pio_nc4.c b/externals/pio2/src/clib/pio_nc4.c deleted file mode 100644 index 735e3755a99..00000000000 --- a/externals/pio2/src/clib/pio_nc4.c +++ /dev/null @@ -1,1078 +0,0 @@ -/** @file - * - * Functions to wrap netCDF-4 functions for PIO. - **/ -#include -#include - -/** - * @ingroup PIO_def_var - * Set deflate (zlib) settings for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable. - * @param shuffle non-zero to turn on shuffle filter (can be good for - * integer data). - * @param deflate non-zero to turn on zlib compression for this - * variable. - * @param deflate_level 1 to 9, with 1 being faster and 9 being more - * compressed. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, - int deflate_level) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_VAR_DEFLATE; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - /* Versions of netCDF 4.4 and earlier do not return an - * error when attempting to turn on deflation with - * parallel I.O. But this is not allowed by HDF5. So - * return the correct error code. */ - ierr = NC_EINVAL; - //ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level); - break; - case PIO_IOTYPE_NETCDF4C: - if (!ios->io_rank) - ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* If there is an error, allocate space for the error string. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check the netCDF return code, and broadcast it to all tasks. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - return ierr; -} - -/** - * @ingroup PIO_inq_var - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * Inquire about deflate (zlib compression) settings for a variable. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param shufflep pointer to an int that will get the status of the - * shuffle filter. - * @param deflatep pointer to an int that will be set to non-zero if - * deflation is in use for this variable. - * @param deflate_levelp pointer to an int that will get the deflation - * level (from 1-9) if deflation is in use for this variable. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, - int *deflatep, int *deflate_levelp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - int ret; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_DEFLATE; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp); - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank == 0) - ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* If there is an error, allocate space for the error string. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check the netCDF return code, and broadcast it to all tasks. */ - ierr = check_netcdf(file, ierr, errstr, __LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - /* Broadcast results to all tasks. */ - if (ierr == PIO_NOERR) - { - if (shufflep) - ierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm); - if (deflatep) - ierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm); - if (deflate_levelp) - ierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm); - } - return ierr; -} - -/** - * @ingroup PIO_def_var - * Set chunksizes for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * Chunksizes have important performance repercussions. NetCDF - * attempts to choose sensible chunk sizes by default, but for best - * performance check chunking against access patterns. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param storage NC_CONTIGUOUS or NC_CHUNKED. - * @param chunksizep an array of chunksizes. Must have a chunksize for - * every variable dimension. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_def_var_chunking(int ncid, int varid, int storage, - const PIO_Offset *chunksizesp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_VAR_CHUNKING; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp); - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank==0) - { - ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp); - } - break; -#endif - case PIO_IOTYPE_NETCDF: - return PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - return PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* Allocate an error string if needed. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check for netCDF error. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - return ierr; -} - -/** - * @ingroup PIO_inq_var - * Inquire about chunksizes for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param storagep pointer to int which will be set to either - * NC_CONTIGUOUS or NC_CHUNKED. - * @param chunksizep pointer to memory where chunksizes will be - * set. There are the same number of chunksizes as there are - * dimensions. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - int ndims; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_CHUNKING; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp); - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank == 0) - { - if ((ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp))) - return ierr; - if ((ierr = nc_inq_varndims(file->fh, varid, &ndims))) - return ierr; - } - break; -#endif - case PIO_IOTYPE_NETCDF: - return PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - return PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* If there is an error, allocate space for the error string. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check the netCDF return code, and broadcast it to all tasks. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error stringif it was allocated. */ - if (errstr != NULL) - free(errstr); - - /* Broadcast results to all tasks. */ - ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm); - if (storagep) - ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm); - if (chunksizesp) - ierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, ios->ioroot, ios->my_comm); - - return ierr; -} - -/** - * @ingroup PIO_def_var - * Set chunksizes for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * Chunksizes have important performance repercussions. NetCDF - * attempts to choose sensible chunk sizes by default, but for best - * performance check chunking against access patterns. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param storage NC_CONTIGUOUS or NC_CHUNKED. - * @param chunksizep an array of chunksizes. Must have a chunksize for - * every variable dimension. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_SET_FILL; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var_fill(file->fh, varid, no_fill, fill_value); - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank==0) - { - ierr = nc_def_var_fill(file->fh, varid, no_fill, fill_value); - } - break; -#endif - case PIO_IOTYPE_NETCDF: - return PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - return PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* Allocate an error string if needed. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check for netCDF error. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - return ierr; -} - -/** - * @ingroup PIO_def_var - * Set chunksizes for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * Chunksizes have important performance repercussions. NetCDF - * attempts to choose sensible chunk sizes by default, but for best - * performance check chunking against access patterns. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param storage NC_CONTIGUOUS or NC_CHUNKED. - * @param chunksizep an array of chunksizes. Must have a chunksize for - * every variable dimension. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_def_var_endian(int ncid, int varid, int endian) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_VAR_ENDIAN; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var_endian(file->fh, varid, endian); - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank==0) - { - ierr = nc_def_var_endian(file->fh, varid, endian); - } - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* Allocate an error string if needed. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check for netCDF error. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - return ierr; -} - -/** - * @ingroup PIO_inq_var - * Inquire about chunksizes for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param storagep pointer to int which will be set to either - * NC_CONTIGUOUS or NC_CHUNKED. - * @param chunksizep pointer to memory where chunksizes will be - * set. There are the same number of chunksizes as there are - * dimensions. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_inq_var_endian(int ncid, int varid, int *endianp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_CHUNKING; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_endian(file->fh, varid, endianp); - break; - case PIO_IOTYPE_NETCDF4C: - if (!ios->io_rank) - ierr = nc_inq_var_endian(file->fh, varid, endianp); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* If there is an error, allocate space for the error string. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check the netCDF return code, and broadcast it to all tasks. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - /* Broadcast results to all tasks. */ - if (endianp) - ierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm); - - return ierr; -} - -/** - * @ingroup PIO_def_var - * - * Set chunk cache netCDF files to be opened/created. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * The file chunk cache for HDF5 can be set, and will apply for any - * files opened or created until the program ends, or the settings are - * changed again. The cache settings apply only to the open file. They - * do not persist with the file, and must be set each time the file is - * opened, before it is opened, if they are to have effect. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param iotype the iotype of files to be created or opened. - * @param size size of file cache. - * @param nelems number of elements in file cache. - * @param preemption preemption setting for file cache. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, - PIO_Offset nelems, float preemption) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - msg = PIO_MSG_SET_CHUNK_CACHE; - - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - /* if (ios->async_interface && ! ios->ioproc){ */ - /* if (ios->compmaster) */ - /* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */ - /* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */ - /* } */ - - switch (iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_set_chunk_cache(size, nelems, preemption); - break; - case PIO_IOTYPE_NETCDF4C: - if (!ios->io_rank) - ierr = nc_set_chunk_cache(size, nelems, preemption); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - - /* Propogate error code to all processes. */ - MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm); - - return ierr; -} - -/** - * @ingroup PIO_def_var - * Get current file chunk cache settings from HDF5. - * - * This function has no effect on netCDF classic files. Calling this - * function with iotype of PIO_IOTYPE_PNETCDF or PIO_IOTYPE_NETCDF - * returns an error. - * - * The file chunk cache for HDF5 can be set, and will apply for any - * files opened or created until the program ends, or the settings are - * changed again. The cache settings apply only to the open file. They - * do not persist with the file, and must be set each time the file is - * opened, before it is opened, if they are to have effect. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * Chunksizes have important performance repercussions. NetCDF - * attempts to choose sensible chunk sizes by default, but for best - * performance check chunking against access patterns. - * - * @param iotype the iotype of files to be created or opened. - * @param sizep gets the size of file cache. - * @param nelemsp gets the number of elements in file cache. - * @param preemptionp gets the preemption setting for file cache. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, - PIO_Offset *nelemsp, float *preemptionp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - /* Since this is a property of the running HDF5 instance, not the - * file, it's not clear if this message passing will apply. For - * now, comment it out. EJH */ - /* msg = PIO_MSG_INQ_VAR_FLETCHER32; */ - - /* if (ios->async_interface && ! ios->ioproc){ */ - /* if (ios->compmaster) */ - /* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */ - /* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */ - /* } */ - - switch (iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp); - break; - case PIO_IOTYPE_NETCDF4C: - if (!ios->io_rank) - ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(iotype,__FILE__,__LINE__); - } - - - /* Check for netCDF error. */ - MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm); - if (!ierr) - { - if (sizep) - if ((ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - ierr = PIO_EIO; - if (nelemsp && !ierr) - if ((ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - ierr = PIO_EIO; - if (preemptionp && !ierr) - if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm))) - ierr = PIO_EIO; - } - - return ierr; -} - -/** - * @ingroup PIO_def_var - * Set chunksizes for a variable. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * Chunksizes have important performance repercussions. NetCDF - * attempts to choose sensible chunk sizes by default, but for best - * performance check chunking against access patterns. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param storage NC_CONTIGUOUS or NC_CHUNKED. - * @param chunksizep an array of chunksizes. Must have a chunksize for - * every variable dimension. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems, - float preemption) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_SET_VAR_CHUNK_CACHE; - - if (ios->async_interface && ! ios->ioproc) - { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption); - break; - case PIO_IOTYPE_NETCDF4C: - if (!ios->io_rank) - ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* Allocate an error string if needed. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check for netCDF error. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - return ierr; -} - -/** - * @ingroup PIO_inq_var - * Get the variable chunk cache settings. - * - * This function only applies to netCDF-4 files. When used with netCDF - * classic files, the error PIO_ENOTNC4 will be returned. - * - * Note that these settings are not part of the data file - they apply - * only to the open file as long as it is open. - * - * See the netCDF - * variable documentation for details about the operation of this - * function. - * - * @param ncid the ncid of the open file. - * @param varid the ID of the variable to set chunksizes for. - * @param sizep will get the size of the cache in bytes. - * @param nelemsp will get the number of elements in the cache. - * @param preemptionp will get the cache preemption value. - * - * @return PIO_NOERR for success, otherwise an error code. - */ -int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp, - float *preemptionp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - - /* Since this is a property of the running HDF5 instance, not the - * file, it's not clear if this message passing will apply. For - * now, comment it out. EJH */ - /* msg = PIO_MSG_INQ_VAR_FLETCHER32; */ - - /* if (ios->async_interface && ! ios->ioproc){ */ - /* if (ios->compmaster) */ - /* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */ - /* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */ - /* } */ - - if (ios->ioproc) - { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_chunk_cache(file->fh, varid, (size_t *)sizep, (size_t *)nelemsp, - preemptionp); - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank == 0) - ierr = nc_get_var_chunk_cache(file->fh, varid, (size_t *)sizep, (size_t *)nelemsp, - preemptionp); - break; -#endif - case PIO_IOTYPE_NETCDF: - ierr = PIO_ENOTNC4; - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = PIO_ENOTNC4; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - /* If there is an error, allocate space for the error string. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check the netCDF return code, and broadcast it to all tasks. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - - /* Broadcast results to all tasks. */ - if (sizep && !ierr) - ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if (nelemsp && !ierr) - ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if (preemptionp && !ierr) - ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm); - - return ierr; -} - - diff --git a/externals/pio2/src/clib/pio_put_nc.c b/externals/pio2/src/clib/pio_put_nc.c deleted file mode 100644 index c0b621eb903..00000000000 --- a/externals/pio2/src/clib/pio_put_nc.c +++ /dev/null @@ -1,5073 +0,0 @@ -#include -#include - -/// -/// PIO interface to nc_put_vars_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_uchar(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_ushort(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_ulonglong(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_uint(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_ushort(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_ushort(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_ushort(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_uchar(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_short (int ncid, int varid, const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_short(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_short(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_short(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_int(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_text(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_ulonglong(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_ulonglong(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_ulonglong(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_int (int ncid, int varid, const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_int(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_int(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_int(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_longlong (int ncid, int varid, const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_longlong(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_longlong(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_schar (int ncid, int varid, const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_schar(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_schar(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_uint(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_uint(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1 -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_double (int ncid, int varid, const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_double(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_double(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_double(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_float (int ncid, int varid, const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_float(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_float(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_float(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_text (int ncid, int varid, const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_text(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_text(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_text(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_longlong(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_double(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_uchar(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_uchar(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_uchar(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_long (int ncid, int varid, const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_long(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_long(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_long(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - diff --git a/externals/pio2/src/clib/pio_rearrange.c b/externals/pio2/src/clib/pio_rearrange.c deleted file mode 100644 index e343e922a00..00000000000 --- a/externals/pio2/src/clib/pio_rearrange.c +++ /dev/null @@ -1,1692 +0,0 @@ -//// -/// @file pio_rearrange.c -/// @author Jim Edwards -/// @date 2014 -/// @brief Code to map IO to model decomposition -/// -/// -/// -/// -/// @see http://code.google.com/p/parallelio/ -//// -#include -#include - -/** internal variable used for debugging */ -int tmpioproc=-1; - - -/** @internal - ** Convert an index into a list of dimensions. E.g., for index 4 into a - ** array defined as a[3][2], will return 1 1. - ** @endinternal -*/ -void idx_to_dim_list(const int ndims, const int gdims[], const PIO_Offset idx, - PIO_Offset dim_list[]) -{ - int i, curr_idx, next_idx; - curr_idx = idx; - // Easiest to start from the right and move left. - for (i = ndims-1; i >= 0; --i) { - // This way of doing div/mod is slightly faster than using "/" and "%". - next_idx = curr_idx / gdims[i]; - dim_list[i] = curr_idx - (next_idx*gdims[i]); - curr_idx = next_idx; - } -} -/** - ** @internal - ** Expand a region along dimension dim, by incrementing count[i] as much as - ** possible, consistent with the map. - ** - ** Once max_size is reached, the map is exhausted, or the next entries fail - ** to match, expand_region updates the count and calls itself with the next - ** outermost dimension, until the region has been expanded as much as - ** possible along all dimensions. - ** @endinternal - */ -void expand_region(const int dim, const int gdims[], const int maplen, - const PIO_Offset map[], const int region_size, - const int region_stride, const int max_size[], - PIO_Offset count[]) -{ - int i, j, test_idx, expansion_done; - // Precondition: maplen >= region_size (thus loop runs at least once). - - // Flag used to signal that we can no longer expand the region along - // dimension dim. - expansion_done = 0; - - // Expand no greater than max_size along this dimension. - for (i = 1; i <= max_size[dim]; ++i) { - // Count so far is at least i. - count[dim] = i; - - // Now see if we can expand to i+1 by checking that the next - // region_size elements are ahead by exactly region_stride. - // Assuming monotonicity in the map, we could skip this for the - // innermost dimension, but it's necessary past that because the - // region does not necessarily comprise contiguous values. - for (j = 0; j < region_size; ++j) { - test_idx = j + i*region_size; - // If we have exhausted the map, or the map no longer matches, - // we are done, break out of both loops. - if (test_idx >= maplen || map[test_idx] != map[j] + i*region_stride) { - expansion_done = 1; - break; - } - } - if (expansion_done) break; - } - - // Move on to next outermost dimension if there are more left, else return. - if (dim > 0) { - expand_region(dim-1, gdims, maplen, map, region_size*count[dim], - region_stride*gdims[dim], max_size, count); - } -} -/** - ** @internal - ** Set start and count so that they describe the first region in map. - ** @endinternal - */ -PIO_Offset find_region(const int ndims, const int gdims[], - const int maplen, const PIO_Offset map[], - PIO_Offset start[], PIO_Offset count[]) -{ - int dim; - int max_size[ndims]; - PIO_Offset regionlen=1; - // Preconditions (which might be useful to check/assert): - // ndims is > 0 - // maplen is > 0 - // all elements of map are inside the bounds specified by gdims - // The map array is 1 based, but calculations are 0 based - idx_to_dim_list(ndims, gdims, map[0]-1, start); - - for (dim = 0; dim < ndims; ++dim) { - // Can't expand beyond the array edge. - max_size[dim] = gdims[dim] - start[dim]; - } - - // For each dimension, figure out how far we can expand in that dimension - // while staying contiguous in the input array. - // - // Start with the innermost dimension (ndims-1), and it will recurse - // through to the outermost dimensions. - expand_region(ndims-1, gdims, maplen, map, 1, 1, max_size, count); - - for(dim=0;dim=0; i--){ - lindex += lcoord[i]*stride; - stride = stride*count[i]; - } - return lindex; - -} - -/** - ** @internal -** Compute the max io buffersize needed for a given variable -** @endinternal -*/ -void compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc) -{ - PIO_Offset iosize, totiosize; - int i; - io_region *region; - - // compute the max io buffer size, for conveneance it is the combined size of all regions - totiosize=0; - region = iodesc->firstregion; - while(region != NULL){ - if(region->count[0]>0){ - iosize=1; - for(i=0;indims;i++) - iosize*=region->count[i]; - totiosize+=iosize; - } - region = region->next; - } - // Share the max io buffer size with all io tasks - CheckMPIReturn(MPI_Allreduce(MPI_IN_PLACE, &totiosize, 1, MPI_OFFSET, MPI_MAX, io_comm),__FILE__,__LINE__); - iodesc->maxiobuflen = totiosize; - if(iodesc->maxiobuflen<=0 ){ - fprintf(stderr,"%s %d %ld %ld %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen,totiosize,MPI_OFFSET,MPI_MAX,io_comm); - piodie("ERROR: maxiobuflen<=0",__FILE__,__LINE__); - } - -} -/** - ** @internal -** Create the derived MPI datatypes used for comp2io and io2comp transfers -** @param basetype The type of data (int,real,double). -** @param msgcnt The number of MPI messages/tasks to use. -** @param dlen The length of the data array. -** @param mindex An array of indexes into the data array from the comp map -** @param mcount The number of indexes to be put on each mpi message/task -** @param *mfrom A pointer to the previous structure in the read/write list -** @param mtype The final data structure sent through MPI to the read/write -** @endinternal -*/ -int create_mpi_datatypes(const MPI_Datatype basetype,const int msgcnt,const PIO_Offset dlen, const PIO_Offset mindex[],const int mcount[], - int *mfrom, MPI_Datatype mtype[]) -{ - PIO_Offset bsizeT[msgcnt]; - int blocksize; - int numinds; - PIO_Offset *lindex = NULL; - - numinds=0; - for(int j=0;j=0,"dlen < 0",__FILE__,__LINE__); - pioassert(numinds>=0, "num inds < 0",__FILE__,__LINE__); - - - if(mindex != NULL){ - // memcpy(lindex, mindex, (size_t) (dlen*sizeof(PIO_Offset))); - lindex = malloc(numinds*sizeof(PIO_Offset)); - memcpy(lindex, mindex, (size_t) (numinds*sizeof(PIO_Offset))); - } - - bsizeT[0]=0; - mtype[0] = MPI_DATATYPE_NULL; - int pos = 0; - int ii = 0; - if(msgcnt>0){ - - if(mfrom == NULL){ - for(int i=0;i0){ - bsizeT[ii] = GCDblocksize(mcount[i], lindex+pos); - ii++; - pos+=mcount[i]; - - } - } - blocksize = (int) lgcd_array(ii ,bsizeT); - }else{ - blocksize=1; - } - - pos = 0; - for(int i=0;i< msgcnt; i++){ - if(mcount[i]>0){ - int len = mcount[i]/blocksize; - int displace[len]; - if(blocksize==1){ - if(mfrom == NULL){ - for(int j=0;jrtype==NULL){ - if(iodesc->nrecvs>0){ - iodesc->rtype = (MPI_Datatype *) bget(iodesc->nrecvs * sizeof(MPI_Datatype)); - if(iodesc->rtype == NULL){ - piomemerror(ios,iodesc->nrecvs * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - for(i=0; inrecvs; i++){ - iodesc->rtype[i] = MPI_DATATYPE_NULL; - } - if(iodesc->rearranger==PIO_REARR_SUBSET){ - create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen, iodesc->rindex, iodesc->rcount, iodesc->rfrom, iodesc->rtype); - }else{ - create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen, iodesc->rindex, iodesc->rcount, NULL, iodesc->rtype); - } - } - /* if(tmpioproc==95) { - MPI_Aint lb; - MPI_Aint extent; - for(i=0;irtype[i], &lb, &extent); - printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,lb,extent); - - } - } - */ - } - } - - - if(iodesc->stype==NULL){ - int ntypes; - int ncnt; - if(iodesc->rearranger==PIO_REARR_SUBSET){ - ntypes = 1; - ncnt = iodesc->scount[0]; - }else{ - ntypes = ios.num_iotasks; - ncnt = iodesc->ndof; - } - - // printf("COMP: %d\n",ntypes); - - - iodesc->stype = (MPI_Datatype *) bget(ntypes * sizeof(MPI_Datatype)); - if(iodesc->stype == NULL){ - piomemerror(ios,ntypes * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - for(i=0; istype[i] = MPI_DATATYPE_NULL; - } - iodesc->num_stypes = ntypes; - - create_mpi_datatypes(iodesc->basetype, ntypes, ncnt, iodesc->sindex, iodesc->scount, NULL, iodesc->stype); - /* if(tmpioproc==95) { - MPI_Aint lb; - MPI_Aint extent; - for(i=0;istype[i], &lb, &extent); - printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,lb,extent); - } - } - */ - } - - return PIO_NOERR; - -} - - -/** - ** @internal -** Completes the mapping for the box rearranger -** @endinternal -*/ - -int compute_counts(const iosystem_desc_t ios, io_desc_t *iodesc, const int maplen, - const int dest_ioproc[], const PIO_Offset dest_ioindex[], MPI_Comm mycomm) -{ - - int i; - int iorank; - - int rank; - int ntasks; - - MPI_Comm_rank(mycomm, &rank); - MPI_Comm_size(mycomm, &ntasks); - - - MPI_Datatype sr_types[ntasks]; - int send_counts[ntasks]; - int send_displs[ntasks]; - int recv_counts[ntasks]; - int recv_displs[ntasks]; - int *recv_buf=NULL; - int nrecvs; - int maxreq = MAX_GATHER_BLOCK_SIZE; - int ierr; - int io_comprank; - int ioindex; - int tsize; - int numiotasks; - PIO_Offset s2rindex[iodesc->ndof]; - - - - if(iodesc->rearranger==PIO_REARR_BOX) - numiotasks = ios.num_iotasks; - else - numiotasks=1; - - iodesc->scount = (int *) bget(numiotasks*sizeof(int)); - if(iodesc->scount == NULL){ - piomemerror(ios,numiotasks * sizeof(int), __FILE__,__LINE__); - } - for(i=0;iscount[i]=0; - - // iodesc->scount is the amount of data sent to each task from the current task - for(i=0;i= 0){ - (iodesc->scount[dest_ioproc[i]])++; - } - } - - // for(i=0;iscount = %d\n",iodesc->scount[i]); - - for(i=0;irearranger==PIO_REARR_SUBSET) - io_comprank=0; - else - io_comprank = ios.ioranks[i]; - send_counts[io_comprank] = 1; - send_displs[io_comprank] = i*sizeof(int); - } - - if(ios.ioproc){ - recv_buf = (int *) bget(ntasks * sizeof(int)); - if(recv_buf == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - for(i=0;iscount[i]); - - // Share the iodesc->scount from each compute task to all io tasks - ierr = pio_swapm( iodesc->scount, send_counts, send_displs, sr_types, - recv_buf, recv_counts, recv_displs, sr_types, - mycomm, false, false, maxreq); - - nrecvs = 0; - if(ios.ioproc){ - // printf("recv_buf = "); - for(i=0;ircount = (int *) bget(max(1,nrecvs)*sizeof(int)); - if(iodesc->rcount == NULL){ - piomemerror(ios,max(1,nrecvs) * sizeof(int), __FILE__,__LINE__); - } - iodesc->rfrom = (int *) bget(max(1,nrecvs)*sizeof(int)); - if(iodesc->rfrom == NULL){ - piomemerror(ios,max(1,nrecvs) * sizeof(int), __FILE__,__LINE__); - } - for(i=0;ircount[i]=0; - iodesc->rfrom[i]=0; - } - - nrecvs = 0; - for(i=0;ircount[nrecvs] = recv_buf[i]; - iodesc->rfrom[nrecvs] = i; - nrecvs++; - } - - } - brel(recv_buf); - } - - iodesc->nrecvs = nrecvs; - if(iodesc->sindex == NULL && iodesc->ndof>0){ - iodesc->sindex = (PIO_Offset *) bget(iodesc->ndof * sizeof(PIO_Offset)); - if(iodesc->sindex == NULL){ - piomemerror(ios,iodesc->ndof * sizeof(PIO_Offset), __FILE__,__LINE__); - } - for(i=0;indof;i++) - iodesc->sindex[i]=0; - } - int tempcount[numiotasks]; - int spos[numiotasks]; - - spos[0]=0; - tempcount[0]=0; - for(i=1;iscount[i-1]; - tempcount[i]=0; - } - - for(i=0;i -1){ - // this should be moved to create_box - if(iodesc->rearranger==PIO_REARR_BOX) - iodesc->sindex[spos[iorank]+tempcount[iorank]] = i; - - s2rindex[spos[iorank]+tempcount[iorank]] = ioindex; - (tempcount[iorank])++; - } - } - // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,iodesc->llen,iodesc->ndof, maplen,spos[0]+tempcount[0]); - - for(i=0;irearranger==PIO_REARR_BOX){ - io_comprank = ios.ioranks[i]; - }else{ - io_comprank=0; - } - send_counts[io_comprank] = iodesc->scount[i]; - if(send_counts[io_comprank]>0) - send_displs[io_comprank] = spos[i]*tsize ; - } - - if(ios.ioproc){ - int totalrecv=0; - for(i=0;irfrom[i]] = iodesc->rcount[i]; - totalrecv+=iodesc->rcount[i]; - } - recv_displs[0] = 0; - for(i=1;irfrom[i]] = recv_displs[iodesc->rfrom[i-1]]+iodesc->rcount[i-1]*tsize; - - - if(totalrecv>0){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,totalrecv,iodesc->llen); - totalrecv = iodesc->llen; // can reduce memory usage here - iodesc->rindex = (PIO_Offset *) bget(totalrecv*sizeof(PIO_Offset)); - if(iodesc->rindex == NULL){ - piomemerror(ios,totalrecv * sizeof(PIO_Offset), __FILE__,__LINE__); - } - for(i=0;irindex[i]=0; - } - } - // printf("%d rbuf_size %d\n",ios.comp_rank,rbuf_size); - - - // s2rindex is the list of indeces on each compute task - /* - printf("%d s2rindex: ", ios.comp_rank); - for(i=0;indof;i++) - printf("%ld ",s2rindex[i]); - printf("\n"); - */ - // printf("%s %d %d %d %d %d %d %d\n",__FILE__,__LINE__,send_counts[0],recv_counts[0],send_displs[0],recv_displs[0],sr_types[0],iodesc->llen); - ierr = pio_swapm( s2rindex, send_counts, send_displs, sr_types, - iodesc->rindex, recv_counts, recv_displs, sr_types, - mycomm, false, false, 0); - // printf("%s %d\n",__FILE__,__LINE__); - - // rindex is an array of the indices of the data to be sent from - // this io task to each compute task. - /* - if(ios.ioproc){ - printf("%d rindex: ",ios.io_rank); - for(int j=0;jllen;j++) - printf(" %ld ",iodesc->rindex[j]); - printf("\n"); - } - */ - return ierr; - -} - -/** - ** @internal - ** Moves data from compute tasks to IO tasks. - ** @endinternal - ** - */ - - -int rearrange_comp2io(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, - void *rbuf, const int nvars) -{ - int ntasks; - int niotasks; - int *scount = iodesc->scount; - int ivar; - int i, tsize; - int *sendcounts; - int *recvcounts; - int *sdispls; - int *rdispls; - MPI_Datatype *sendtypes; - MPI_Datatype *recvtypes; - MPI_Comm mycomm; - - // maxreq = 0; - // printf("%s %d %d %d\n",__FILE__,__LINE__,nvars,iodesc->ndof); -#ifdef TIMING - GPTLstart("PIO:rearrange_comp2io"); -#endif - - if(iodesc->rearranger == PIO_REARR_BOX){ - mycomm = ios.union_comm; - niotasks = ios.num_iotasks; - }else{ - mycomm = iodesc->subset_comm; - niotasks = 1; - } - MPI_Comm_size(mycomm, &ntasks); - - pioassert(nvars>0,"nvars must be > 0",__FILE__,__LINE__); - MPI_Type_size(iodesc->basetype, &tsize); - - define_iodesc_datatypes(ios, iodesc); - - sendcounts = (int *) bget(ntasks*sizeof(int)); - if(sendcounts == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - recvcounts = (int *) bget(ntasks*sizeof(int)); - if(recvcounts == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - sdispls = (int *) bget(ntasks*sizeof(int)); - if(sdispls == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - rdispls = (int *) bget(ntasks*sizeof(int)); - if(rdispls == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - sendtypes = (MPI_Datatype *) bget(ntasks*sizeof(MPI_Datatype)); - if(sendtypes == NULL){ - piomemerror(ios,ntasks * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - recvtypes = (MPI_Datatype *) bget(ntasks*sizeof(MPI_Datatype)); - if(recvtypes == NULL){ - piomemerror(ios,ntasks * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - //printf("%s %d %ld %ld\n",__FILE__,__LINE__,sendtypes,recvtypes); - for(i=0;inrecvs>0){ - // printf("%d: rindex[%d] %d\n",ios.comp_rank,0,iodesc->rindex[0]); - for( i=0;inrecvs;i++){ - if(iodesc->rtype[i] != MPI_DATATYPE_NULL){ - if(iodesc->rearranger==PIO_REARR_SUBSET){ - recvcounts[ i ] = 1; - // The stride here is the length of the collected array (llen) - MPI_Type_hvector(nvars, 1, (MPI_Aint) iodesc->llen*tsize, iodesc->rtype[i], recvtypes+i); - if(recvtypes[i] == MPI_DATATYPE_NULL){ - piodie("Unexpected NULL MPI DATATYPE",__FILE__,__LINE__); - } - MPI_Type_commit(recvtypes+i); - //recvtypes[ i ] = iodesc->rtype[i]; - }else{ - recvcounts[ iodesc->rfrom[i] ] = 1; - MPI_Type_hvector(nvars, 1,(MPI_Aint) iodesc->llen*tsize,iodesc->rtype[i], recvtypes+iodesc->rfrom[i]); - if(recvtypes[iodesc->rfrom[i]] == MPI_DATATYPE_NULL){ - piodie("Unexpected NULL MPI DATATYPE",__FILE__,__LINE__); - } - MPI_Type_commit(recvtypes+iodesc->rfrom[i]); - // recvtypes[ iodesc->rfrom[i] ] = iodesc->rtype[i]; - rdispls[ iodesc->rfrom[i] ] = 0; - // printf("%d: rindex[%d] %d\n",ios.comp_rank,0,iodesc->rindex[0]); - } - } - // printf("%d: rindex[%d] %d\n",ios.comp_rank,i,iodesc->rindex[i]); - - } - } - - for( i=0;irearranger==PIO_REARR_SUBSET) - io_comprank=0; - - // printf("scount[%d]=%d\n",i,scount[i]); - if(scount[i] > 0 && sbuf != NULL) { - sendcounts[io_comprank]=1; - MPI_Type_hvector(nvars, 1, (MPI_Aint) iodesc->ndof*tsize, iodesc->stype[i], sendtypes+io_comprank); - if(sendtypes[io_comprank] == MPI_DATATYPE_NULL){ - piodie("Unexpected NULL MPI DATATYPE",__FILE__,__LINE__); - } - MPI_Type_commit(sendtypes+io_comprank); - }else{ - sendcounts[io_comprank]=0; - } - } - // printf("%s %d %d\n",__FILE__,__LINE__,((int *)sbuf)[5]); -//printf("%s %d %ld %ld %ld %ld\n",__FILE__,__LINE__,sendtypes[0],recvtypes,sbuf,rbuf); - // Data in sbuf on the compute nodes is sent to rbuf on the ionodes - - // for(i=0;indof;i++) - // printf("%s %d %d %d \n",__FILE__,__LINE__,i,((int *)sbuf)[i]); - - pio_swapm( sbuf, sendcounts, sdispls, sendtypes, - rbuf, recvcounts, rdispls, recvtypes, - mycomm, iodesc->handshake, iodesc->isend, iodesc->max_requests); - // if(ios.ioproc) - // for(i=0;illen;i++) - // printf("%s %d %d %d\n",__FILE__,__LINE__,i,((int *)rbuf)[i]); - //printf("%s %d %ld %ld\n",__FILE__,__LINE__,sendtypes[0],recvtypes); - - brel(sendcounts); - brel(recvcounts); - brel(sdispls); - brel(rdispls); -//printf("%s %d %ld %ld\n",__FILE__,__LINE__,sendtypes[0],recvtypes); - for( i=0;iscount; - - int *sendcounts; - int *recvcounts; - int *sdispls; - int *rdispls; - MPI_Datatype *sendtypes; - MPI_Datatype *recvtypes; - - int i, tsize; -#ifdef TIMING - GPTLstart("PIO:rearrange_io2comp"); -#endif - if(iodesc->rearranger==PIO_REARR_BOX){ - mycomm = ios.union_comm; - niotasks = ios.num_iotasks; - }else{ - mycomm = iodesc->subset_comm; - niotasks=1; - } - MPI_Comm_size(mycomm, &ntasks); - - define_iodesc_datatypes(ios, iodesc); - - sendcounts = (int *) bget(ntasks*sizeof(int)); - if(sendcounts == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - recvcounts = (int *) bget(ntasks*sizeof(int)); - if(recvcounts == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - sdispls = (int *) bget(ntasks*sizeof(int)); - if(sdispls == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - rdispls = (int *) bget(ntasks*sizeof(int)); - if(rdispls == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - sendtypes = (MPI_Datatype *) bget(ntasks*sizeof(MPI_Datatype)); - if(sendtypes == NULL){ - piomemerror(ios,ntasks * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - recvtypes = (MPI_Datatype *) bget(ntasks*sizeof(MPI_Datatype)); - if(recvtypes == NULL){ - piomemerror(ios,ntasks * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - - for( i=0;i< ntasks;i++){ - sendcounts[i] = 0; - recvcounts[i] = 0; - sdispls[i] = 0; - rdispls[i] = 0; - sendtypes[ i ] = MPI_DATATYPE_NULL; - recvtypes[ i ] = MPI_DATATYPE_NULL; - } - if(ios.ioproc){ - for( i=0;i< iodesc->nrecvs;i++){ - if(iodesc->rtype[i] != MPI_DATATYPE_NULL){ - if(iodesc->rearranger==PIO_REARR_SUBSET){ - if(sbuf != NULL){ - sendcounts[ i ] = 1; - sendtypes[ i ] = iodesc->rtype[i]; - } - }else{ - sendcounts[ iodesc->rfrom[i] ] = 1; - sendtypes[ iodesc->rfrom[i] ] = iodesc->rtype[i]; - } - } - } - } - - for( i=0;irearranger==PIO_REARR_SUBSET) - io_comprank=0; - if(scount[i] > 0 && iodesc->stype[i] != MPI_DATATYPE_NULL) { - recvcounts[io_comprank]=1; - recvtypes[io_comprank]=iodesc->stype[i]; - } - } - - // - // Data in sbuf on the ionodes is sent to rbuf on the compute nodes - // - #if DEBUG - printf("%s %d \n",__FILE__,__LINE__); - #endif - pio_swapm( sbuf, sendcounts, sdispls, sendtypes, - rbuf, recvcounts, rdispls, recvtypes, - mycomm, iodesc->handshake, iodesc->isend, iodesc->max_requests); - #if DEBUG - printf("%s %d \n",__FILE__,__LINE__); - #endif - brel(sendcounts); - brel(recvcounts); - brel(sdispls); - brel(rdispls); - brel(sendtypes); - brel(recvtypes); - -#ifdef TIMING - GPTLstop("PIO:rearrange_io2comp"); -#endif - - return PIO_NOERR; - -} - -void determine_fill(iosystem_desc_t ios, io_desc_t *iodesc, const int gsize[], const PIO_Offset compmap[]) -{ - PIO_Offset totalllen=0; - PIO_Offset totalgridsize=1; - int i; - for( i=0;indims;i++){ - totalgridsize *= gsize[i]; - } - if(iodesc->rearranger==PIO_REARR_SUBSET){ - totalllen=iodesc->llen; - }else{ - for(i=0;indof;i++){ - if(compmap[i]>0){ - totalllen++; - } - } - } - // printf("%s %d %ld %ld\n",__FILE__,__LINE__,totalllen,totalgridsize); - - MPI_Allreduce(MPI_IN_PLACE, &totalllen, 1, PIO_OFFSET, MPI_SUM, ios.union_comm); - - - if(totalllen < totalgridsize){ - //if(ios.iomaster) printf("%s %d %ld %ld\n",__FILE__,__LINE__,totalllen,totalgridsize); - iodesc->needsfill = true; - }else{ - iodesc->needsfill = false; - iodesc->gsize = NULL; - } - // TURN OFF FILL for timing test - // iodesc->needsfill=false; - - if(iodesc->needsfill){ - iodesc->gsize = (PIO_Offset *) bget(iodesc->ndims * sizeof(PIO_Offset)); - if(iodesc->gsize==NULL){ - piomemerror(ios,iodesc->ndims * sizeof(MPI_Datatype), __FILE__,__LINE__); - } - for (i=0;indims;i++){ - iodesc->gsize[i] = gsize[i]; - } - } - - -} - - -void iodesc_dump(io_desc_t *iodesc) -{ - printf("ioid= %d\n",iodesc->ioid); - printf("async_id= %d\n",iodesc->async_id); - printf("nrecvs= %d\n",iodesc->nrecvs); - printf("ndof= %d\n",iodesc->ndof); - printf("ndims= %d\n",iodesc->ndims); - printf("num_aiotasks= %d\n",iodesc->num_aiotasks); - printf("rearranger= %d\n",iodesc->rearranger); - printf("maxregions= %d\n",iodesc->maxregions); - printf("needsfill= %d\n",(int) iodesc->needsfill); - - printf("llen= %ld\n",iodesc->llen); - printf("maxiobuflen= %d\n",iodesc->maxiobuflen); - printf("gsize="); - for(int i=0; i< iodesc->ndims; i++){ - printf(" %d",iodesc->ioid); - } - printf("\n"); - - printf("rindex= "); - for(int j=0;jllen;j++) - printf(" %ld ",iodesc->rindex[j]); - printf("\n"); - - - -} - -/** - ** @internal - ** The box rearranger computes a mapping between IO tasks and compute tasks such that the data - ** on io tasks can be written with a single call to the underlying netcdf library. This - ** may involve an all to all rearrangement in the mapping, but should minimize data movement in - ** lower level libraries - ** @endinternal - ** - */ - -int box_rearrange_create(const iosystem_desc_t ios,const int maplen, const PIO_Offset compmap[], const int gsize[], - const int ndims, io_desc_t *iodesc) -{ - int ierr=PIO_NOERR; - int nprocs = ios.num_comptasks; - int nioprocs = ios.num_iotasks; - PIO_Offset gstride[ndims]; - PIO_Offset iomap; - PIO_Offset start[ndims], count[ndims]; - int tsize, i, j, k, llen; - MPI_Datatype dtype; - int dest_ioproc[maplen]; - PIO_Offset dest_ioindex[maplen]; - int sndlths[nprocs]; - int sdispls[nprocs]; - int recvlths[nprocs]; - int rdispls[nprocs]; - MPI_Datatype dtypes[nprocs]; - PIO_Offset iomaplen[nioprocs]; - int maxreq = MAX_GATHER_BLOCK_SIZE; - - iodesc->rearranger = PIO_REARR_BOX; - - iodesc->ndof = maplen; - gstride[ndims-1]=1; - for(int i=ndims-2;i>=0; i--) - gstride[i]=gstride[i+1]*gsize[i+1]; - - MPI_Type_size(MPI_OFFSET, &tsize); - - for(i=0; i< maplen; i++){ - dest_ioproc[i] = -1; - dest_ioindex[i] = -1; - } - for(i=0;illen=0; - if(ios.ioproc){ - for( i=0;illen=1; - for(i=0;illen *= iodesc->firstregion->count[i]; - } - determine_fill(ios, iodesc, gsize, compmap); - - /* - if(ios.ioproc){ - for(i=0; ifirstregion->start[i],iodesc->firstregion->count[i]); - } - printf("\n%s %d\n",__FILE__,__LINE__); - } - */ - - for( i=0;illen), sndlths, sdispls, dtypes, - iomaplen, recvlths, rdispls, dtypes, - ios.union_comm, false, false, maxreq); - - - - - /* - printf("%s %d %d\n",__FILE__,__LINE__,nioprocs); - for(i=0; i0){ - int io_comprank = ios.ioranks[i]; - for( j=0; jfirstregion->count, sndlths, sdispls, dtypes, - count, recvlths, rdispls, dtypes, - ios.union_comm, false, false, maxreq); - - // The start from iotask i is sent to all compute tasks - pio_swapm(iodesc->firstregion->start, sndlths, sdispls, dtypes, - start, recvlths, rdispls, dtypes, - ios.union_comm, false, false, maxreq); - - for(k=0;k= start[j] && gcoord[j] < start[j]+count[j]){ - lcoord[j] = gcoord[j] - start[j]; - }else{ - found = false; - break; - } - } - if(found){ - dest_ioindex[k] = coord_to_lindex(ndims, lcoord, count); - dest_ioproc[k] = i; - } - } - } - } - - for(k=0; k0){ - fprintf(stderr,"No destination found for compmap[%d] = %ld\n",k,compmap[k]); - piodie(" ",__FILE__,__LINE__); - } - } - - compute_counts(ios, iodesc, maplen, dest_ioproc, dest_ioindex, ios.union_comm); - - if(ios.ioproc){ - compute_maxIObuffersize(ios.io_comm, iodesc); - } - compute_maxaggregate_bytes(ios, iodesc); - - -#ifdef DEBUG - iodesc_dump(iodesc); -#endif - return PIO_NOERR; -} -/** - ** @internal - ** compare offsets is used by the sort in the subset rearrange - ** @endinternal - */ -int compare_offsets(const void *a,const void *b) -{ - mapsort *x = (mapsort *) a; - mapsort *y = (mapsort *) b; - return (int) (x->iomap - y->iomap); -} - -/** - ** @internal - ** Each region is a block of output which can be represented in a single call to the underlying - ** netcdf library. This can be as small as a single data point, but we hope we've aggragated better than that. - ** @endinternal - */ -void get_start_and_count_regions(const int ndims, const int gdims[],const int maplen, - const PIO_Offset map[], int *maxregions, io_region *firstregion) -{ - int i; - int nmaplen; - int regionlen; - io_region *region, *prevregion; - - nmaplen = 0; - region = firstregion; - if(map != NULL){ - while(map[nmaplen++]<=0); - nmaplen--; - } - region->loffset=nmaplen; - - *maxregions = 1; - prevregion=NULL; - - while(nmaplen < maplen){ - // Here we find the largest region from the current offset into the iomap - // regionlen is the size of that region and we step to that point in the map array - // until we reach the end - for(i=0;icount[i]=1; - } - - regionlen = find_region(ndims, gdims, maplen-nmaplen, - map+nmaplen, region->start, region->count); - // printf("%s %d %d %d\n",__FILE__,__LINE__,region->start[0],region->count[0]); - pioassert(region->start[0]>=0,"failed to find region",__FILE__,__LINE__); - - nmaplen = nmaplen+regionlen; - if(region->next==NULL && nmaplen< maplen){ - region->next = alloc_region(ndims); - // The offset into the local array buffer is the sum of the sizes of all of the previous regions (loffset) - region=region->next; - region->loffset = nmaplen; - // The calls to the io library are collective and so we must have the same number of regions on each - // io task maxregions will be the total number of regions on this task - (*maxregions)++; - } - } - - -} - -/** - ** @internal - ** The subset rearranger needs a mapping from compute tasks to IO task, the only requirement is - ** that each compute task map to one and only one IO task. This mapping groups by mpi task id - ** others are possible and may be better for certain decompositions - ** @endinternal - ** - */ -void default_subset_partition(const iosystem_desc_t ios, io_desc_t *iodesc) -{ - int taskratio = ios.num_comptasks/ios.num_iotasks; - int color; - int key; - - /* Create a new comm for each subset group with the io task in rank 0 and - only 1 io task per group */ - - if(ios.ioproc){ - key = 0; - color= ios.io_rank; - }else{ - key=max(1,ios.comp_rank%taskratio+1); - color = min(ios.num_iotasks-1,ios.comp_rank/taskratio); - } - - MPI_Comm_split(ios.comp_comm, color, key, &(iodesc->subset_comm)); - -} - -/** - ** @internal - ** The subset rearranger computes a mapping between IO tasks and compute tasks such that each compute - ** task communicates with one and only one IO task. - ** @endinternal - ** - */ - -int subset_rearrange_create(const iosystem_desc_t ios,const int maplen, PIO_Offset compmap[], - const int gsize[], const int ndims, io_desc_t *iodesc) -{ - - int taskratio; - - int i, j, jlast; - bool hs=false; - bool isend=false; - PIO_Offset *iomap=NULL; - int ierr = PIO_NOERR; - mapsort *map=NULL; - PIO_Offset totalgridsize; - PIO_Offset *srcindex=NULL; - PIO_Offset *myfillgrid = NULL; - int maxregions; - int maxreq = MAX_GATHER_BLOCK_SIZE; - int rank, ntasks, rcnt; - size_t pio_offset_size=sizeof(PIO_Offset); - - - tmpioproc = ios.io_rank; - - - /* subset partitions each have exactly 1 io task which is task 0 of that subset_comm */ - /* TODO: introduce a mechanism for users to define partitions */ - default_subset_partition(ios, iodesc); - iodesc->rearranger = PIO_REARR_SUBSET; - - MPI_Comm_rank(iodesc->subset_comm, &rank); - MPI_Comm_size(iodesc->subset_comm, &ntasks); - - if(ios.ioproc){ - pioassert(rank==0,"Bad io rank in subset create",__FILE__,__LINE__); - }else{ - pioassert(rank>0 && rankndof = maplen; - if(ios.ioproc){ - iodesc->rcount = (int *) bget(ntasks *sizeof(int)); - if(iodesc->rcount == NULL){ - piomemerror(ios,ntasks * sizeof(int), __FILE__,__LINE__); - } - rcnt = 1; - } - iodesc->scount = (int *) bget(sizeof(int)); - if(iodesc->scount == NULL){ - piomemerror(ios,sizeof(int), __FILE__,__LINE__); - } - iodesc->scount[0]=0; - totalgridsize=1; - for( i=0;i=0 && compmap[i]<=totalgridsize, "Compmap value out of bounds",__FILE__,__LINE__); - if(compmap[i]>0){ - (iodesc->scount[0])++; - } - } - if(iodesc->scount[0]>0){ - iodesc->sindex = (PIO_Offset *) bget(iodesc->scount[0]*pio_offset_size); - if(iodesc->sindex == NULL){ - piomemerror(ios,iodesc->scount[0]*pio_offset_size, __FILE__,__LINE__); - } - for(i=0;iscount[0];i++) - iodesc->sindex[i]=0; - } - - j=0; - for(i=0;i0){ - iodesc->sindex[j++]=i; - - } - } - - // Pass the reduced maplen (without holes) from each compute task to its associated IO task - // printf("%s %d %ld\n",__FILE__,__LINE__,iodesc->scount); - - pio_fc_gather( (void *) iodesc->scount, 1, MPI_INT, - (void *) iodesc->rcount, rcnt, MPI_INT, - 0, iodesc->subset_comm, maxreq); - - iodesc->llen = 0; - - int rdispls[ntasks]; - int recvlths[ntasks]; - - if(ios.ioproc){ - for(i=0;illen+=iodesc->rcount[i]; - rdispls[i]=0; - recvlths[i]= iodesc->rcount[ i ]; - if(i>0) - rdispls[i] = rdispls[i-1]+ iodesc->rcount[ i-1 ]; - } - // printf("%s %d %ld %d %d\n",__FILE__,__LINE__,iodesc,iodesc->llen,maplen); - - if(iodesc->llen>0){ - srcindex = (PIO_Offset *) bget(iodesc->llen*pio_offset_size); - if(srcindex == NULL){ - piomemerror(ios,iodesc->llen * pio_offset_size, __FILE__,__LINE__); - } - for(i=0;i < iodesc->llen; i++) - srcindex[i]=0; - } - }else{ - for(i=0;isindex, iodesc->scount[0], PIO_OFFSET, - (void *) srcindex, recvlths, rdispls, PIO_OFFSET, - 0, iodesc->subset_comm, maxreq); - - if(ios.ioproc && iodesc->llen>0){ - map = (mapsort *) bget(iodesc->llen * sizeof(mapsort)); - if(map == NULL){ - piomemerror(ios,iodesc->llen * sizeof(mapsort), __FILE__,__LINE__); - } - iomap = (PIO_Offset *) bget(iodesc->llen*pio_offset_size); - if(iomap == NULL){ - piomemerror(ios,iodesc->llen * pio_offset_size, __FILE__,__LINE__); - } - for(i=0;illen;i++) - iomap[i]=0; - } - - // Now pass the compmap, skipping the holes - - PIO_Offset *shrtmap; - if(maplen>iodesc->scount[0] && iodesc->scount[0]>0){ - shrtmap = (PIO_Offset *) bget(iodesc->scount[0]*pio_offset_size); - if(shrtmap == NULL){ - piomemerror(ios,iodesc->scount[0] * pio_offset_size, __FILE__,__LINE__); - } - for(i=0;iscount[0];i++) - shrtmap[i]=0; - - j=0; - for(i=0;i0) - shrtmap[j++]=compmap[i]; - }else{ - shrtmap = compmap; - } - - pio_fc_gatherv((void *) shrtmap, iodesc->scount[0], PIO_OFFSET, - (void *) iomap, recvlths, rdispls, PIO_OFFSET, - 0, iodesc->subset_comm, maxreq); - - - if(shrtmap != compmap) - brel(shrtmap); - - if(ios.ioproc && iodesc->llen>0){ - int pos=0; - int k=0; - mapsort *mptr; - for(i=0;ircount[i];j++){ - mptr = map+k; - mptr->rfrom = i; - mptr->soffset = srcindex[pos+j]; - mptr->iomap = iomap[pos+j]; - k++; - } - pos += iodesc->rcount[i]; - } - // sort the mapping, this will transpose the data into IO order - qsort(map, iodesc->llen, sizeof(mapsort), compare_offsets); - - iodesc->rindex = (PIO_Offset *) bget(iodesc->llen*pio_offset_size); - if(iodesc->rindex == NULL){ - piomemerror(ios,iodesc->llen * pio_offset_size, __FILE__,__LINE__); - } - iodesc->rfrom = (int *) bget(iodesc->llen*sizeof(int)); - if(iodesc->rfrom == NULL){ - piomemerror(ios,iodesc->llen * sizeof(int), __FILE__,__LINE__); - } - for(i=0;illen;i++){ - iodesc->rindex[i]=0; - iodesc->rfrom[i]=0; - } - } - int cnt[ntasks]; - int sndlths[ntasks]; - int sdispls[ntasks]; - MPI_Datatype dtypes[ntasks]; - for(i=0;iscount[0]; - mapsort *mptr; - for(i=0;illen;i++){ - mptr = map+i; - iodesc->rfrom[i] = mptr->rfrom; - iodesc->rindex[i]=i; - iomap[i] = mptr->iomap; - srcindex[ (cnt[iodesc->rfrom[i]])++ ]=mptr->soffset; - } - - if(ios.ioproc && iodesc->needsfill){ - /* we need the list of offsets which are not in the union of iomap */ - PIO_Offset thisgridsize[ios.num_iotasks]; - PIO_Offset thisgridmin[ios.num_iotasks], thisgridmax[ios.num_iotasks]; - int nio; - PIO_Offset *myusegrid = NULL; - int gcnt[ios.num_iotasks]; - int displs[ios.num_iotasks]; - - thisgridmin[0] = 1; - thisgridsize[0] = totalgridsize/ios.num_iotasks; - thisgridmax[0] = thisgridsize[0]; - int xtra = totalgridsize - thisgridsize[0]*ios.num_iotasks; - - for(nio=0;nio0){ - thisgridsize[nio] = totalgridsize/ios.num_iotasks; - if(nio>=ios.num_iotasks-xtra) thisgridsize[nio]++; - thisgridmin[nio]=thisgridmax[nio-1]+1; - thisgridmax[nio]= thisgridmin[nio] + thisgridsize[nio]-1; - } - for(int i=0;illen;i++){ - // printf("%s %d %d %d %ld %d %d\n",__FILE__,__LINE__,i,nio,iomap[i],thisgridmin[nio],thisgridmax[nio]); - if(iomap[i]>=thisgridmin[nio] && iomap[i]<=thisgridmax[nio]){ - cnt++; - if(cnt==1) imin=i; - } - } - // printf("%s %d %d %d %d\n",__FILE__,__LINE__,nio,cnt,imin); - MPI_Gather(&cnt, 1, MPI_INT, gcnt, 1, MPI_INT, nio, ios.io_comm); - if(nio==ios.io_rank){ - displs[0]=0; - // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,nio,0,gcnt[0],displs[0]); - for(i=1;i=0){ - grid[j]=1; - cnt++; - } - } - if(myusegrid!=NULL){ - brel(myusegrid); - } - iodesc->holegridsize=thisgridsize[ios.io_rank]-cnt; - if(iodesc->holegridsize>0){ - myfillgrid = (PIO_Offset *) bget(iodesc->holegridsize * sizeof(PIO_Offset)); - } - for(i=0;iholegridsize;i++){ - myfillgrid[i]=-1; - } - - j=0; - for(i=0;imaxfillregions=0; - if(myfillgrid!=NULL){ - iodesc->fillregion = alloc_region(iodesc->ndims); - get_start_and_count_regions(iodesc->ndims,gsize,iodesc->holegridsize, myfillgrid,&(iodesc->maxfillregions), - iodesc->fillregion); - brel(myfillgrid); - maxregions = iodesc->maxfillregions; - } - MPI_Allreduce(MPI_IN_PLACE,&maxregions,1, MPI_INT, MPI_MAX, ios.io_comm); - iodesc->maxfillregions = maxregions; - } - - CheckMPIReturn(MPI_Scatterv((void *) srcindex, recvlths, rdispls, PIO_OFFSET, - (void *) iodesc->sindex, iodesc->scount[0], PIO_OFFSET, - 0, iodesc->subset_comm),__FILE__,__LINE__); - - - - if(ios.ioproc){ - - /* - printf("%s %d %d\n",__FILE__,__LINE__,ios.io_rank); - for(i=0;illen;i++) - printf("%d ",iomap[i]); - printf("\n"); - */ - iodesc->maxregions = 0; - get_start_and_count_regions(iodesc->ndims,gsize,iodesc->llen, iomap,&(iodesc->maxregions), - iodesc->firstregion); - maxregions = iodesc->maxregions; - MPI_Allreduce(MPI_IN_PLACE,&maxregions,1, MPI_INT, MPI_MAX, ios.io_comm); - iodesc->maxregions = maxregions; - if(iomap != NULL) - brel(iomap); - - - if(map != NULL) - brel(map); - - if(srcindex != NULL) - brel(srcindex); - - compute_maxIObuffersize(ios.io_comm, iodesc); - - iodesc->nrecvs=ntasks; -#ifdef DEBUG - iodesc_dump(iodesc); -#endif - - } - /* using maxiobuflen compute the maximum number of vars of this type that the io - task buffer can handle */ - - compute_maxaggregate_bytes(ios, iodesc); - - - return ierr; - - -} - - - -void performance_tune_rearranger(iosystem_desc_t ios, io_desc_t *iodesc) -{ - bool handshake; - bool isend; - int nreqs; - int maxreqs; - int nprocs; - MPI_Comm mycomm; -#ifdef TIMING -#ifdef PERFTUNE - double *wall, usr[2], sys[2]; - void *cbuf, *ibuf; - int tsize; - int myrank; - - MPI_Type_size(iodesc->basetype, &tsize); - cbuf = NULL; - ibuf = NULL; - if(iodesc->ndof>0){ - cbuf = bget( iodesc->ndof * tsize ); - } - if(iodesc->llen>0){ - ibuf = bget( iodesc->llen * tsize ); - } - - if(iodesc->rearranger == PIO_REARR_BOX){ - mycomm = ios.union_comm; - }else{ - mycomm = iodesc->subset_comm; - } - - MPI_Comm_size(mycomm, &nprocs); - MPI_Comm_rank(mycomm, &myrank); - - int log2 = log(nprocs) / log(2) + 1; - wall = bget(2*4*log2*sizeof(double)); - double mintime; - int k=0; - - MPI_Barrier(mycomm); - GPTLstamp( &wall[0], &usr[0], &sys[0]); - rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1); - rearrange_io2comp(ios, iodesc, ibuf, cbuf); - GPTLstamp( &wall[1], &usr[1], &sys[1]); - mintime = wall[1]-wall[0]; - MPI_Allreduce(MPI_IN_PLACE, &mintime, 1, MPI_DOUBLE, MPI_MAX, mycomm); - - handshake = iodesc->handshake; - isend = iodesc->isend; - maxreqs = iodesc->max_requests; - - for(int i=0; i<2; i++){ - if(i==0){ - iodesc->handshake = false; - }else{ - iodesc->handshake = true; - } - for(int j=0; j<2; j++){ - if(j==0){ - iodesc->isend = false; - }else{ - iodesc->isend = true; - } - iodesc->max_requests = 0; - - for(nreqs=nprocs;nreqs>=2;nreqs/=2){ - iodesc->max_requests = nreqs; - - // if(myrank==0){ - // printf("%s %d %d %d %d %f\n",__FILE__,__LINE__,nreqs,handshake,isend,mintime); - // } - MPI_Barrier(mycomm); - GPTLstamp( wall, usr, sys); - rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1); - rearrange_io2comp(ios, iodesc, ibuf, cbuf); - GPTLstamp( wall+1, usr, sys); - wall[1]-=wall[0]; - MPI_Allreduce(MPI_IN_PLACE, wall+1,1,MPI_DOUBLE, MPI_MAX, mycomm); - - if(wall[1] < mintime*0.95){ - handshake = iodesc->handshake; - isend = iodesc->isend; - maxreqs = nreqs; - mintime = wall[1]; - }else if(wall[1]> (mintime*1.05)){ - exit; - } - } - } - } - - - iodesc->handshake = handshake; - iodesc->isend = isend; - iodesc->max_requests = maxreqs; - if(myrank==0){ - printf("spmd optimization: maxreqs: %d handshake:%d isend:%d mintime=%f\n",maxreqs,handshake,isend,mintime); - } - brel(wall); - brel(cbuf); - brel(ibuf); -#endif -#endif - -} diff --git a/externals/pio2/src/clib/pio_spmd.c b/externals/pio2/src/clib/pio_spmd.c deleted file mode 100644 index 9645ed45b0f..00000000000 --- a/externals/pio2/src/clib/pio_spmd.c +++ /dev/null @@ -1,650 +0,0 @@ -/** - * @file pio_spmd.c - * @author Algorithms modeled after spmd_utils in the Community Atmosphere Model; C translation Jim Edwards - * @date 2014 - * @brief MPI_Gather, MPI_Gatherv, and MPI_Alltoallw with flow control options - */ - -#ifdef TESTSWAPM -#include -#include -#include -#include -#include -#define PIO_NOERR 0 -#define min(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a < _b ? _a : _b; }) -#define MAX_GATHER_BLOCK_SIZE 32 -#else -#include -#include -#endif - -/** - ** @brief Wrapper for MPI calls to print the Error string on error - */ -void CheckMPIReturn(const int ierr,const char file[],const int line) -{ - - if(ierr != MPI_SUCCESS){ - char errstring[MPI_MAX_ERROR_STRING]; - int errstrlen; - int mpierr = MPI_Error_string( ierr, errstring, &errstrlen); - - fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n",errstring, file, line); - - } -} - - -/** - ** @brief Provides the functionality of MPI_Gather with flow control options - */ - -int pio_fc_gather( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnt, const MPI_Datatype recvtype, const int root, - const MPI_Comm comm, const int flow_cntl) -{ - bool fc_gather; - int gather_block_size; - int mytask, nprocs; - int mtag; - MPI_Status status; - int ierr; - int hs; - int displs; - int dsize; - - - - if(flow_cntl > 0){ - fc_gather = true; - gather_block_size = min(flow_cntl,MAX_GATHER_BLOCK_SIZE); - }else{ - fc_gather = false; - } - - if(fc_gather){ - CheckMPIReturn(MPI_Comm_rank (comm, &mytask), __FILE__,__LINE__); - CheckMPIReturn(MPI_Comm_size (comm, &nprocs), __FILE__,__LINE__); - - mtag = 2*nprocs; - hs = 1; - - if(mytask == root){ - int preposts = min(nprocs-1, gather_block_size); - int head=0; - int count=0; - int tail = 0; - MPI_Request rcvid[gather_block_size]; - - CheckMPIReturn(MPI_Type_size(recvtype, &dsize), __FILE__,__LINE__); - - for(int p=0;p 0){ - count++; - if(count > preposts){ - CheckMPIReturn(MPI_Wait(rcvid+tail, &status), __FILE__,__LINE__); - tail = (tail+1) % preposts; - } - displs = p*recvcnt*dsize; - - char *ptr = (char *) recvbuf + displs; - - CheckMPIReturn(MPI_Irecv( ptr, recvcnt, recvtype, p, mtag, comm, rcvid+head), __FILE__,__LINE__); - head= (head+1) % preposts; - CheckMPIReturn(MPI_Send( &hs, 1, MPI_INT, p, mtag, comm), __FILE__,__LINE__); - } - } - } - - // copy local data - CheckMPIReturn(MPI_Type_size(sendtype, &dsize), __FILE__,__LINE__); - memcpy(recvbuf, sendbuf, sendcnt*dsize ); - - count = min(count, preposts); - - if(count>0){ - CheckMPIReturn(MPI_Waitall( count, rcvid, MPI_STATUSES_IGNORE),__FILE__,__LINE__); - } - }else{ - if(sendcnt > 0){ - CheckMPIReturn(MPI_Recv( &hs, 1, MPI_INT, root, mtag, comm, &status), __FILE__,__LINE__); - CheckMPIReturn(MPI_Send( sendbuf, sendcnt, sendtype, root, mtag, comm), __FILE__,__LINE__); - } - } - }else{ - // printf("%s %d %ld %d %ld %d %d\n",__FILE__,__LINE__,sendbuf,sendcnt,recvbuf,recvcnt,root); - CheckMPIReturn(MPI_Gather ( sendbuf, sendcnt, sendtype, recvbuf, recvcnt, recvtype, root, comm), __FILE__,__LINE__); - } - - return PIO_NOERR; -} - - - -/** - ** @brief Provides the functionality of MPI_Gatherv with flow control options - */ - -int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnts[], const int displs[], - const MPI_Datatype recvtype, const int root, - const MPI_Comm comm, const int flow_cntl) -{ - bool fc_gather; - int gather_block_size; - int mytask, nprocs; - int mtag; - MPI_Status status; - int ierr; - int hs; - int dsize; - - - if(flow_cntl > 0){ - fc_gather = true; - gather_block_size = min(flow_cntl,MAX_GATHER_BLOCK_SIZE); - }else{ - fc_gather = false; - } - - if(fc_gather){ - CheckMPIReturn(MPI_Comm_rank (comm, &mytask), __FILE__,__LINE__); - CheckMPIReturn(MPI_Comm_size (comm, &nprocs), __FILE__,__LINE__); - - mtag = 2*nprocs; - hs = 1; - - if(mytask == root){ - int preposts = min(nprocs-1, gather_block_size); - int head=0; - int count=0; - int tail = 0; - MPI_Request rcvid[gather_block_size]; - // printf("%s %d %d\n",__FILE__,__LINE__,(int) recvtype); - CheckMPIReturn(MPI_Type_size(recvtype, &dsize), __FILE__,__LINE__); - - for(int p=0;p 0){ - count++; - if(count > preposts){ - CheckMPIReturn(MPI_Wait(rcvid+tail, &status), __FILE__,__LINE__); - tail = (tail+1) % preposts; - } - - void *ptr = (void *)((char *) recvbuf + dsize*displs[p]); - - // printf("%s %d %d %d\n",__FILE__,__LINE__,p,(int) recvtype); - CheckMPIReturn(MPI_Irecv( ptr, recvcnts[p], recvtype, p, mtag, comm, rcvid+head), __FILE__,__LINE__); - head= (head+1) % preposts; - CheckMPIReturn(MPI_Send( &hs, 1, MPI_INT, p, mtag, comm), __FILE__,__LINE__); - } - } - } - - // copy local data - CheckMPIReturn(MPI_Type_size(sendtype, &dsize), __FILE__,__LINE__); - CheckMPIReturn(MPI_Sendrecv(sendbuf, sendcnt, sendtype, - mytask, 102, recvbuf, recvcnts[mytask], recvtype, - mytask, 102, comm, &status),__FILE__,__LINE__); - - count = min(count, preposts); - if(count>0) - CheckMPIReturn(MPI_Waitall( count, rcvid, MPI_STATUSES_IGNORE),__FILE__,__LINE__); - }else{ - if(sendcnt > 0){ - CheckMPIReturn(MPI_Recv( &hs, 1, MPI_INT, root, mtag, comm, &status), __FILE__,__LINE__); - CheckMPIReturn(MPI_Send( sendbuf, sendcnt, sendtype, root, mtag, comm), __FILE__,__LINE__); - } - } - }else{ - CheckMPIReturn(MPI_Gatherv ( sendbuf, sendcnt, sendtype, recvbuf, recvcnts, displs, recvtype, root, comm), __FILE__,__LINE__); - } - - return PIO_NOERR; -} - -/// -/// @brief Returns the smallest power of 2 greater than i -/// -int ceil2(const int i) -{ - int p=1; - while(p np-1)? -1: q; - return pair; -} - -/** - ** @brief Provides the functionality of MPI_Alltoallw with flow control options - */ -int pio_swapm(void *sndbuf, int sndlths[], int sdispls[], MPI_Datatype stypes[], - void *rcvbuf, int rcvlths[], int rdispls[], MPI_Datatype rtypes[], - MPI_Comm comm,const bool handshake, bool isend,const int max_requests) -{ - - int nprocs; - int mytask; - - int maxsend=0; - int maxrecv=0; - - CheckMPIReturn(MPI_Comm_size(comm, &nprocs),__FILE__,__LINE__); - CheckMPIReturn(MPI_Comm_rank(comm, &mytask),__FILE__,__LINE__); - - - if(max_requests == 0) { -#ifdef DEBUG - int totalrecv=0; - int totalsend=0; - for(int i=0;i 0){ - void *sptr, *rptr; - int extent, lb; - tag = mytask + offset_t; - sptr = (void *)((char *) sndbuf + sdispls[mytask]); - rptr = (void *)((char *) rcvbuf + rdispls[mytask]); - - /* - MPI_Type_get_extent(stypes[mytask], &lb, &extent); - printf("%s %d %d %d\n",__FILE__,__LINE__,extent, lb); - MPI_Type_get_extent(rtypes[mytask], &lb, &extent); - printf("%s %d %d %d\n",__FILE__,__LINE__,extent, lb); - */ -#ifdef ONEWAY - CheckMPIReturn(MPI_Sendrecv(sptr, sndlths[mytask],stypes[mytask], - mytask, tag, rptr, rcvlths[mytask], rtypes[mytask], - mytask, tag, comm, &status),__FILE__,__LINE__); -#else - // printf("%s %d \n",__FILE__,__LINE__); - CheckMPIReturn(MPI_Irecv(rptr, rcvlths[mytask], rtypes[mytask], - mytask, tag, comm, rcvids),__FILE__,__LINE__); - //printf("%s %d \n",__FILE__,__LINE__); - CheckMPIReturn(MPI_Send(sptr, sndlths[mytask], stypes[mytask], - mytask, tag, comm),__FILE__,__LINE__); - - //printf("%s %d %d\n",__FILE__,__LINE__,rcvids[0]); - CheckMPIReturn(MPI_Wait(rcvids, &status),__FILE__,__LINE__); - -#endif - - - } - if(nprocs==1) - return PIO_NOERR; - - int swapids[nprocs]; - MPI_Request sndids[nprocs]; - MPI_Request hs_rcvids[nprocs]; - - for(int i=0;i= 0 && (sndlths[p] > 0 || rcvlths[p] > 0)){ - swapids[steps++] = p; - } - } - - if(steps == 1){ - maxreq = 1; - maxreqh = 1; - }else{ - if(max_requests > 1 && max_requests=steps){ - maxreq = steps; - maxreqh = steps; - }else{ - maxreq = 2; - maxreqh = 1; - } - } - if(handshake){ - hs = 1; - for(istep=0; istep 0){ - tag = mytask+offset_t; - CheckMPIReturn(MPI_Irecv( &hs, 1, MPI_INT, p, tag, comm, hs_rcvids+istep), __FILE__,__LINE__); - } - } - } - for(istep=0;istep < maxreq; istep++){ - p = swapids[istep]; - if(rcvlths[p] > 0){ - tag = p + offset_t; - ptr = (void *)((char *) rcvbuf + rdispls[p]); - - // printf("%s %d %d %d\n",__FILE__,__LINE__,p,(int) rtypes[p]); - CheckMPIReturn(MPI_Irecv( ptr, rcvlths[p], rtypes[p], p, tag, comm, rcvids+istep), __FILE__,__LINE__); - - if(handshake) - CheckMPIReturn(MPI_Send( &hs, 1, MPI_INT, p, tag, comm), __FILE__,__LINE__); - } - } - - rstep = maxreq; - for(istep = 0; istep < steps; istep++){ - p = swapids[istep]; - if(sndlths[p] > 0){ - tag = mytask + offset_t; - if(handshake){ - CheckMPIReturn(MPI_Wait ( hs_rcvids+istep, &status), __FILE__,__LINE__); - hs_rcvids[istep] = MPI_REQUEST_NULL; - } - ptr = (void *)((char *) sndbuf + sdispls[p]); - - if(isend){ - CheckMPIReturn(MPI_Irsend(ptr, sndlths[p], stypes[p], p, tag, comm,sndids+istep), __FILE__,__LINE__); - }else{ - CheckMPIReturn(MPI_Send(ptr, sndlths[p], stypes[p], p, tag, comm), __FILE__,__LINE__); - } - - } - if(istep > maxreqh){ - p = istep - maxreqh; - if(rcvids[p] != MPI_REQUEST_NULL){ - CheckMPIReturn(MPI_Wait(rcvids+p, &status), __FILE__,__LINE__); - rcvids[p] = MPI_REQUEST_NULL; - } - if(rstep < steps){ - p = swapids[rstep]; - if(handshake && sndlths[p] > 0){ - tag = mytask + offset_t; - CheckMPIReturn(MPI_Irecv( &hs, 1, MPI_INT, p, tag, comm, hs_rcvids+rstep), __FILE__,__LINE__); - } - if(rcvlths[p] > 0){ - tag = p + offset_t; - - ptr = (void *)((char *) rcvbuf + rdispls[p]); - CheckMPIReturn(MPI_Irecv( ptr, rcvlths[p], rtypes[p], p, tag, comm, rcvids+rstep), __FILE__,__LINE__); - if(handshake) - CheckMPIReturn(MPI_Send( &hs, 1, MPI_INT, p, tag, comm), __FILE__,__LINE__); - } - rstep++; - } - } - } - // printf("%s %d %d \n",__FILE__,__LINE__,nprocs); - if(steps>0){ - CheckMPIReturn(MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE), __FILE__,__LINE__); - if(isend) - CheckMPIReturn(MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE), __FILE__,__LINE__); - } - // printf("%s %d %d \n",__FILE__,__LINE__,nprocs); - - return PIO_NOERR; -} - -#ifdef TESTSWAPM_OLD -#include -/* -This program tests MPI_Alltoallw by having processor i send different -amounts of data to each processor. -The first test sends i items to processor i from all processors. -*/ -int main( int argc, char **argv ) -{ - MPI_Comm comm; - int *sbuf, *rbuf; - int rank, size; - int *sendcounts, *recvcounts, *rdispls, *sdispls; - int i, j, *p, err; - MPI_Datatype *sendtypes, *recvtypes; - struct timeval t1, t2; - int msg_cnt; - - MPI_Init( &argc, &argv ); - err = 0; - comm = MPI_COMM_WORLD; - /* Create the buffer */ - MPI_Comm_size( comm, &size ); - MPI_Comm_rank( comm, &rank ); - sbuf = (int *)malloc( size * size * sizeof(int) ); - rbuf = (int *)malloc( size * size * sizeof(int) ); - if (!sbuf || !rbuf) { - fprintf( stderr, "Could not allocated buffers!\n" ); - fflush(stderr); - MPI_Abort( comm, 1 ); - } - /* Test pio_fc_gather */ - - msg_cnt=0; - while(msg_cnt<= MAX_GATHER_BLOCK_SIZE){ - /* Load up the buffers */ - for (i=0; i -#include - - -static int counter=0; - -/** - ** @brief Check to see if PIO has been initialized. - */ -int PIOc_iosystem_is_active(const int iosysid, bool *active) -{ - iosystem_desc_t *ios; - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - if(ios->comp_comm == MPI_COMM_NULL && ios->io_comm == MPI_COMM_NULL){ - *active = false; - }else{ - *active = true; - } - return PIO_NOERR; -} -/** - ** @brief Check to see if PIO file is open. - */ - -int PIOc_File_is_Open(int ncid) -{ - file_desc_t *file; - file = pio_get_file_from_id(ncid); - if(file==NULL) - return 0; - else - return 1; -} - -/** - ** @brief Set the error handling method to be used for subsequent - ** pio library calls, returns the previous method setting - */ -int PIOc_Set_File_Error_Handling(int ncid, int method) -{ - file_desc_t *file; - int oldmethod; - file = pio_get_file_from_id(ncid); - oldmethod = file->iosystem->error_handler; - file->iosystem->error_handler = method; - return(oldmethod); -} - -/** - ** @brief Increment the unlimited dimension of the given variable - */ -int PIOc_advanceframe(int ncid, int varid) -{ - file_desc_t *file; - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - - file->varlist[varid].record++; - - return(PIO_NOERR); -} - -/** - * @ingroup PIO_setframe - * @brief Set the unlimited dimension of the given variable - * - * @param ncid the ncid of the file. - * @param varid the varid of the variable - * @param frame the value of the unlimited dimension. In c 0 for the - * first record, 1 for the second - * - * @return PIO_NOERR for no error, or error code. - */ -int PIOc_setframe(const int ncid, const int varid, const int frame) -{ - file_desc_t *file; - file = pio_get_file_from_id(ncid); - if(file == NULL || varid<0 || varid>=PIO_MAX_VARS){ - return PIO_EBADID; - } - - file->varlist[varid].record = frame; - - return(PIO_NOERR); -} - -/** - ** @brief Get the number of IO tasks set. - */ -int PIOc_get_numiotasks(int iosysid, int *numiotasks) -{ - iosystem_desc_t *ios; - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - *numiotasks = ios->num_iotasks; - - return PIO_NOERR; - -} - - -/** - ** @brief Get the IO rank on the current task - */ -int PIOc_get_iorank(int iosysid, int *iorank) -{ - iosystem_desc_t *ios; - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - *iorank = ios->io_rank; - - return PIO_NOERR; - -} - -/** - ** @brief Get the local size of the variable - */ - -int PIOc_get_local_array_size(int ioid) -{ - io_desc_t *iodesc; - iodesc = pio_get_iodesc_from_id(ioid); - return(iodesc->ndof); -} - -/** - ** @ingroup PIO_error_method - ** @brief Set the error handling method used for subsequent calls - */ - - int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method) -{ - iosystem_desc_t *ios; - int oldmethod; - ios = pio_get_iosystem_from_id(iosysid); - if(ios==NULL){ - fprintf(stderr,"%s %d Error setting eh method\n",__FILE__,__LINE__); - print_trace(stderr); - return PIO_EBADID; - } - oldmethod = ios->error_handler; - ios->error_handler = method; - return(oldmethod); -} - -/** - ** @ingroup PIO_initdecomp - ** @brief C interface to the initdecomp - ** @param iosysid @copydoc iosystem_desc_t (input) - ** @param basetype the basic PIO data type used (input) - ** @param ndims the number of dimensions in the variable (input) - ** @param dims[] the global size of each dimension (input) - ** @param maplen the local length of the compmap array (input) - ** @param compmap[] a 1 based array of offsets into the array record on file. A 0 in this array indicates a value which should not be transfered. (input) - ** @param ioidp the io description pointer (output) - ** @param rearranger the rearranger to be used for this decomp or NULL to use the default (optional input) - ** @param iostart An optional array of start values for block cyclic decompositions (optional input) - ** @param iocount An optional array of count values for block cyclic decompositions (optional input) - */ - - -int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[], - const int maplen, const PIO_Offset *compmap, int *ioidp,const int *rearranger, - const PIO_Offset *iostart,const PIO_Offset *iocount) -{ - iosystem_desc_t *ios; - io_desc_t *iodesc; - int mpierr; - int ierr; - int iosize; - int ndisp; - - - - for(int i=0;inum_comptasks < 100) { - sprintf(filename, "piodecomp%2.2dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter); - }else if(ios->num_comptasks < 10000) { - sprintf(filename, "piodecomp%4.4dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter); - }else{ - sprintf(filename, "piodecomp%6.6dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter); - } - PIOc_writemap(filename,ndims,dims,maplen,compmap,ios->comp_comm); - counter++; - } - - - iodesc = malloc_iodesc(basetype, ndims); - if(rearranger == NULL) - iodesc->rearranger = ios->default_rearranger; - else - iodesc->rearranger = *rearranger; - - if(iodesc->rearranger==PIO_REARR_SUBSET){ - if((iostart != NULL) && (iocount != NULL)){ - fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp", - "are incompatable with subset rearrange method and will be ignored"); - } - iodesc->num_aiotasks = ios->num_iotasks; - ierr = subset_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc); - }else{ - if(ios->ioproc){ - // Unless the user specifies the start and count for each IO task compute it. - if((iostart != NULL) && (iocount != NULL)){ - // printf("iocount[0] = %ld %ld\n",iocount[0], iocount); - iodesc->maxiobuflen=1; - for(int i=0;ifirstregion->start[i] = iostart[i]; - iodesc->firstregion->count[i] = iocount[i]; - compute_maxIObuffersize(ios->io_comm, iodesc); - - } - iodesc->num_aiotasks = ios->num_iotasks; - }else{ - iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims, - ios->num_iotasks, ios->io_rank, - iodesc->firstregion->start, iodesc->firstregion->count); - } - compute_maxIObuffersize(ios->io_comm, iodesc); - - } - // Depending on array size and io-blocksize the actual number of io tasks used may vary - CheckMPIReturn(MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, - ios->my_comm),__FILE__,__LINE__); - // Compute the communications pattern for this decomposition - if(iodesc->rearranger==PIO_REARR_BOX){ - ierr = box_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc); - } - /* - if(ios->ioproc){ - io_region *ioregion = iodesc->firstregion; - while(ioregion != NULL){ - for(int i=0;istart[i],ioregion->count[i]); - ioregion = ioregion->next; - } - } - */ - } - - *ioidp = pio_add_to_iodesc_list(iodesc); - - performance_tune_rearranger(*ios, iodesc); - - return PIO_NOERR; -} - -/** - ** @ingroup PIO_initdecomp - ** This is a simplified initdecomp which can be used if the memory order of the data can be - ** expressed in terms of start and count on the file. - ** in this case we compute the compdof and use the subset rearranger - */ - - -int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, const int dims[], - const long int start[], const long int count[], int *ioidp) - -{ - iosystem_desc_t *ios; - io_desc_t *iodesc; - int mpierr; - int ierr; - int iosize; - int ndisp; - - - for(int i=0;idims[i]){ - piodie("Invalid start or count argument ",__FILE__,__LINE__); - } - } - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - int n, i, maplen=1; - - for( i=0;i=0;n--){ - prod[n]=prod[n+1]*dims[n+1]; - loc[n]=0; - } - for(i=0;i=0;n--){ - compmap[i]+=(start[n]+loc[n])*prod[n]; - } - n=ndims-1; - loc[n]=(loc[n]+1)%count[n]; - while(loc[n]==0 && n>0){ - n--; - loc[n]=(loc[n]+1)%count[n]; - } - } - int rearr = PIO_REARR_SUBSET; - PIOc_InitDecomp( iosysid, basetype,ndims, dims, - maplen, compmap, ioidp, &rearr, NULL, NULL); - - - return PIO_NOERR; -} - - -/** - ** @ingroup PIO_init - ** @brief library initialization used when IO tasks are a subset of compute tasks - ** @param comp_comm the MPI_Comm of the compute tasks - ** @param num_iotasks the number of io tasks to use - ** @param stride the offset between io tasks in the comp_comm - ** @param base the comp_comm index of the first io task - ** @param rearr the rearranger to use by default, this may be overriden in the @ref PIO_initdecomp - ** @param iosysidp index of the defined system descriptor - */ - -int PIOc_Init_Intracomm(const MPI_Comm comp_comm, - const int num_iotasks, const int stride, - const int base,const int rearr, int *iosysidp) -{ - iosystem_desc_t *iosys; - int ierr = PIO_NOERR; - int ustride; - int lbase; - int mpierr; - - iosys = (iosystem_desc_t *) malloc(sizeof(iosystem_desc_t)); - - /* Copy the computation communicator into union_comm. */ - mpierr = MPI_Comm_dup(comp_comm, &iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Copy the computation communicator into comp_comm. */ - if (!ierr) - { - mpierr = MPI_Comm_dup(comp_comm, &iosys->comp_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - } - - if (!ierr) - { - iosys->my_comm = iosys->comp_comm; - iosys->io_comm = MPI_COMM_NULL; - iosys->intercomm = MPI_COMM_NULL; - iosys->error_handler = PIO_INTERNAL_ERROR; - iosys->async_interface= false; - iosys->compmaster = false; - iosys->iomaster = false; - iosys->ioproc = false; - iosys->default_rearranger = rearr; - iosys->num_iotasks = num_iotasks; - - ustride = stride; - - /* Find MPI rank and number of tasks in comp_comm communicator. */ - CheckMPIReturn(MPI_Comm_rank(iosys->comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__); - CheckMPIReturn(MPI_Comm_size(iosys->comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__); - if(iosys->comp_rank==0) - iosys->compmaster = true; - - /* Ensure that settings for number of computation tasks, number - * of IO tasks, and the stride are reasonable. */ - if((iosys->num_comptasks == 1) && (num_iotasks*ustride > 1)) { - // This is a serial run with a bad configuration. Set up a single task. - fprintf(stderr, "PIO_TP PIOc_Init_Intracomm reset stride and tasks.\n"); - iosys->num_iotasks = 1; - ustride = 1; - } - if((iosys->num_iotasks < 1) || (((iosys->num_iotasks-1)*ustride+1) > iosys->num_comptasks)){ - fprintf(stderr, "PIO_TP PIOc_Init_Intracomm error\n"); - fprintf(stderr, "num_iotasks=%d, ustride=%d, num_comptasks=%d\n", num_iotasks, ustride, iosys->num_comptasks); - return PIO_EBADID; - } - - /* Create an array that holds the ranks of the tasks to be used for IO. */ - iosys->ioranks = (int *) calloc(sizeof(int), iosys->num_iotasks); - for(int i=0;i< iosys->num_iotasks; i++){ - iosys->ioranks[i] = (base + i*ustride) % iosys->num_comptasks; - if(iosys->ioranks[i] == iosys->comp_rank) - iosys->ioproc = true; - } - iosys->ioroot = iosys->ioranks[0]; - - /* Create an MPI info object. */ - CheckMPIReturn(MPI_Info_create(&(iosys->info)),__FILE__,__LINE__); - iosys->info = MPI_INFO_NULL; - - if(iosys->comp_rank == iosys->ioranks[0]) - iosys->iomaster = true; - - /* Create a group for the computation tasks. */ - CheckMPIReturn(MPI_Comm_group(iosys->comp_comm, &(iosys->compgroup)),__FILE__,__LINE__); - - /* Create a group for the IO tasks. */ - CheckMPIReturn(MPI_Group_incl(iosys->compgroup, iosys->num_iotasks, iosys->ioranks, - &(iosys->iogroup)),__FILE__,__LINE__); - - /* Create an MPI communicator for the IO tasks. */ - CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__); - - /* For the tasks that are doing IO, get their rank. */ - if(iosys->ioproc) - CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__); - else - iosys->io_rank = -1; - - iosys->union_rank = iosys->comp_rank; - - /* Add this iosys struct to the list in the PIO library. */ - *iosysidp = pio_add_to_iosystem_list(iosys); - - pio_get_env(); - - /* allocate buffer space for compute nodes */ - compute_buffer_init(*iosys); - } - - return ierr; -} - -/** - ** @internal - ** interface to call from pio_init from fortran - ** @endinternal - */ -int PIOc_Init_Intracomm_from_F90(int f90_comp_comm, - const int num_iotasks, const int stride, - const int base, const int rearr, int *iosysidp){ - return PIOc_Init_Intracomm(MPI_Comm_f2c(f90_comp_comm), num_iotasks, stride,base,rearr, iosysidp); -} - -/** - ** @brief Send a hint to the MPI-IO library - ** - */ -int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]) -{ - iosystem_desc_t *ios; - - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - if(ios->ioproc) - CheckMPIReturn( MPI_Info_set(ios->info, hint, hintval), __FILE__,__LINE__); - - return PIO_NOERR; - -} - -/** @ingroup PIO_finalize - * @brief Clean up data structures and exit the pio library. - * - * @param iosysid: the io system ID provided by PIOc_Init_Intracomm(). - * - * @returns 0 for success or non-zero for error. - */ - -int PIOc_finalize(const int iosysid) -{ - iosystem_desc_t *ios, *nios; - - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - /* FIXME: The memory for ioranks is allocated in C only for intracomms - * Remove this check once mem allocs for ioranks completely moves to the - * C code - */ - if(ios->intercomm == MPI_COMM_NULL){ - if(ios->ioranks != NULL){ - free(ios->ioranks); - } - } - - free_cn_buffer_pool(*ios); - - /* Free the MPI groups. */ - MPI_Group_free(&(ios->compgroup)); - MPI_Group_free(&(ios->iogroup)); - - /* Free the MPI communicators. */ - if(ios->intercomm != MPI_COMM_NULL){ - MPI_Comm_free(&(ios->intercomm)); - } - if(ios->io_comm != MPI_COMM_NULL){ - MPI_Comm_free(&(ios->io_comm)); - } - if(ios->comp_comm != MPI_COMM_NULL){ - MPI_Comm_free(&(ios->comp_comm)); - } - if(ios->union_comm != MPI_COMM_NULL){ - MPI_Comm_free(&(ios->union_comm)); - } - - return pio_delete_iosystem_from_list(iosysid); - - -} - -/** - ** @brief return a logical indicating whether this task is an iotask - */ -int PIOc_iam_iotask(const int iosysid, bool *ioproc) -{ - iosystem_desc_t *ios; - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - *ioproc = ios->ioproc; - return PIO_NOERR; -} - -/** - ** @brief return the rank of this task in the io comm or - ** -1 if this task is not in the comm - */ -int PIOc_iotask_rank(const int iosysid, int *iorank) -{ - iosystem_desc_t *ios; - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - *iorank = ios->io_rank; - - return PIO_NOERR; - -} - -/** - ** @brief return true if this iotype is supported in the build, 0 otherwise - */ - -int PIOc_iotype_available(const int iotype) -{ - - switch(iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: - return(1); -#endif - case PIO_IOTYPE_NETCDF: - return(1); -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - return(1); - break; -#endif - default: - return(0); - } - -} diff --git a/externals/pio2/src/clib/pioc_sc.c b/externals/pio2/src/clib/pioc_sc.c deleted file mode 100644 index baf86b4442f..00000000000 --- a/externals/pio2/src/clib/pioc_sc.c +++ /dev/null @@ -1,444 +0,0 @@ -/** - * @file pioc_sc.c - * @author Jim Edwards - * @date 2014 - * @brief Compute start and count arrays for the box rearranger - * - * - * - * - * @see http://code.google.com/p/parallelio/ - */ -#ifdef TESTCALCDECOMP -enum PIO_DATATYPE{ - PIO_REAL, - PIO_INT, - PIO_DOUBLE -}; -#define PIO_NOERR 0 -typedef long long PIO_Offset; -#define max(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a > _b ? _a : _b; }) - -#define min(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a < _b ? _a : _b; }) -#include -#include -#include -#else -#include -#include -#endif - -/** - ** The default target blocksize for each io task when the box rearranger is used. - */ -//#define default_blocksize 884736 -#define default_blocksize 1024 -//#define default_blocksize 388816 -/** - ** The target blocksize for each io task when the box rearranger is used. - */ -int blocksize = default_blocksize; - -/** - ** @ingroup PIO_set_blocksize - ** @brief Set the target blocksize for the box rearranger. - ** - */ -int PIOc_set_blocksize(const int newblocksize) -{ - if(newblocksize > 0) - blocksize = newblocksize; - return PIO_NOERR; -} - - - -/* Recursive Standard C Function: Greatest Common Divisor */ - -int gcd (int a,int b ) -{ - if ( a==0 ) return b; - return gcd ( b%a, a ); -} - -long long lgcd (long long a,long long b ) -{ - if ( a==0 ) return b; - return lgcd ( b%a, a ); -} - - -int gcd_array(int nain, int *ain){ - int i; - int bsize=1; - - for(i=0;i 1){ - bsize = gcd(bsize,ain[i]); - i++; - } - - return bsize; -} - -long long lgcd_array(int nain, long long*ain){ - int i; - long long bsize=1; - - for(i=0;i 1){ - bsize = gcd(bsize,ain[i]); - i++; - } - - return bsize; -} - - -int calcdisplace(const int bsize, const int numblocks,const PIO_Offset map[], int displace[]) -{ - for(int i=0;i= ioprocs-irank){ - lkount++; - if((adds = irank+remainder-ioprocs)>0) - lstart+= adds; - } - *start = lstart; - *kount = lkount; - -} - -PIO_Offset GCDblocksize(const int arrlen, const PIO_Offset arr_in[]){ - // PIO_Offset del_arr[arrlen-1]; - // PIO_Offset loc_arr[arrlen-1]; - PIO_Offset *gaps=NULL; - PIO_Offset *blk_len=NULL; - int i, j, k, n, numblks, numtimes, ii, numgaps; - PIO_Offset bsize, bsizeg, blklensum; - // PIO_Offset *del_arr = (PIO_Offset *) calloc((arrlen-1),sizeof(PIO_Offset)); - // PIO_Offset *loc_arr = (PIO_Offset *) calloc((arrlen-1),sizeof(PIO_Offset)); - PIO_Offset del_arr[arrlen-1]; - PIO_Offset loc_arr[arrlen-1]; - - // printf("%s %d\n",__FILE__,__LINE__); - - numblks=0; - numgaps=0; - numtimes=0; - - for(i=0;i0 && del_arr[i-1]>1){ - return(1); - } - // printf("%s %d %d %d %d\n",__FILE__,__LINE__,i,del_arr[i],arr_in[i]); - - } - } - - - - numblks = numtimes+1; - if(numtimes==0) - n = numblks; - else - n = numtimes; - - // printf("%s %d %d %d\n",__FILE__,__LINE__,numblks,numtimes); - bsize = (PIO_Offset) arrlen; - if(numblks>1){ - PIO_Offset blk_len[numblks]; - PIO_Offset gaps[numtimes]; - // blk_len = (PIO_Offset*) calloc(numblks,sizeof(PIO_Offset)); - // loc_arr = (PIO_Offset*) malloc((arrlen-1)*sizeof(PIO_Offset)); - //printf("%s %d %d\n",__FILE__,__LINE__,numtimes); - if(numtimes>0){ - // gaps = (PIO_Offset *) calloc(numtimes,sizeof(PIO_Offset)); - ii=0; - // printf("%s %d %d\n",__FILE__,__LINE__,arrlen); - for(i=0;i1) - gaps[ii++] = del_arr[i] - 1; - } - numgaps=ii; - } - // printf("%s %d\n",__FILE__,__LINE__); - - j=0; - for(i=0;i 0) { - bsizeg = lgcd_array(numgaps, gaps); - bsize = lgcd(bsize,bsizeg); - // free(gaps); - } - if(arr_in[0]>0) - bsize = lgcd(bsize,arr_in[0]); - // printf("%s %d\n",__FILE__,__LINE__); - - // free(loc_arr); - // free(del_arr); - // free(blk_len); - } - // printf("%s %d\n",__FILE__,__LINE__); - return bsize; -} - - - - - -int CalcStartandCount(const int basetype, const int ndims,const int *gdims, const int num_io_procs, - const int myiorank, PIO_Offset *start, PIO_Offset *kount) -{ - int minbytes; - int maxbytes; - int minblocksize; - int basesize; - int use_io_procs; - int i; - long int p; - long int pgdims; - bool converged; - int iorank; - int ldims; - int tiorank; - int ioprocs; - int tioprocs; - int mystart[ndims], mykount[ndims]; - long int pknt; - long int tpsize=0; - - minbytes = blocksize-256; - maxbytes = blocksize+256; - - switch (basetype){ - case PIO_INT: - basesize = sizeof(int); - break; - case PIO_REAL: - basesize = sizeof(float); - break; - case PIO_DOUBLE: - basesize = sizeof(double); - break; - default: -#ifndef TESTCALCDECOMP - piodie("Invalid basetype ",__FILE__,__LINE__); -#endif - break; - } - minblocksize = minbytes/basesize; - - pgdims = 1; - for( i=0;i=0;i--){ - p=p*gdims[i]; - if(p/use_io_procs > maxbytes){ - ldims = i; - break; - } - } - - if(gdims[ldims]< use_io_procs){ - if(ldims>0 && gdims[ldims-1] > use_io_procs) - ldims--; - else - use_io_procs -= (use_io_procs % gdims[ldims]); - } - - ioprocs = use_io_procs; - tiorank = iorank; -#ifdef TESTCALCDECOMP - if(myiorank==0) printf("%d use_io_procs %d ldims %d\n",__LINE__,use_io_procs,ldims); -#endif - for(i=0;i<=ldims;i++){ - if(gdims[i]>= ioprocs){ - computestartandcount(gdims[i],ioprocs,tiorank,start+i,kount+i); -#ifdef TESTCALCDECOMP - if(myiorank==0) printf("%d tiorank %d i %d start %d count %d\n",__LINE__,tiorank,i,start[i],kount[i]); -#endif - if(start[i]+kount[i]>gdims[i]+1){ -#ifndef TESTCALCDECOMP - piodie("Start plus count exceeds dimension bound",__FILE__,__LINE__); -#endif - } - }else if(gdims[i]>1){ - tioprocs=gdims[i]; - tiorank = (iorank*tioprocs)/ioprocs; - computestartandcount(gdims[i],tioprocs,tiorank,start+i,kount+i); - ioprocs = ioprocs/tioprocs; - tiorank = iorank % ioprocs; - } - - } - if(myiorank==iorank){ - for(i=0;i=pgdims) { - break; - } - } - if(! converged) { -#ifdef TESTCALCDECOMP - printf("%d start %d %d count %d %d tpsize %ld\n",__LINE__,mystart[0],mystart[1],mykount[0],mykount[1],tpsize); -#endif - tpsize = 0; - use_io_procs--; - } - - } - - if(myiorank -#include - -#include -#define versno 2001 - -static pio_swapm_defaults swapm_defaults; -bool PIO_Save_Decomps=false; -/** - ** @brief Get PIO environment variables - ** - **/ -void pio_get_env(void) -{ - char *envptr; - extern bufsize PIO_CNBUFFER_LIMIT; - envptr = getenv("PIO_Save_Decomps"); - - if(envptr != NULL && (strcmp(envptr,"true")==0)){ - PIO_Save_Decomps=true; - } - swapm_defaults.nreqs = 0; - swapm_defaults.handshake=false; - swapm_defaults.isend=false; - - envptr = getenv("PIO_SWAPM"); - if(envptr != NULL){ - char *token = strtok(envptr, ":"); - - swapm_defaults.nreqs = atoi(token); - - token = strtok(NULL, ":"); - - if((token!=NULL) && strcmp(token,"t")==0){ - swapm_defaults.handshake = true; - } - token = strtok(NULL, ":"); - - if((token!=NULL) && strcmp(token,"t")==0){ - swapm_defaults.isend = true; - } - //printf("nreqs %d handshake %d isend %d\n",swapm_defaults.nreqs, swapm_defaults.handshake, swapm_defaults.isend); - } - envptr = getenv("PIO_CNBUFFER_LIMIT"); - if(envptr != NULL){ - int mult=1; - if(strchr(envptr,"M") != NULL){ - mult = 1000000; - }else if(strchr(envptr,"K") != NULL){ - mult = 1000; - } - PIO_CNBUFFER_LIMIT=(bufsize) atoll(envptr)*mult; - - } - - - -} - - - -/* Obtain a backtrace and print it to stderr. */ -void print_trace (FILE *fp) -{ - void *array[10]; - size_t size; - char **strings; - size_t i; - - if(fp==NULL) - fp = stderr; - - size = backtrace (array, 10); - strings = backtrace_symbols (array, size); - - fprintf (fp,"Obtained %zd stack frames.\n", size); - - for (i = 0; i < size; i++) - fprintf (fp,"%s\n", strings[i]); - - free (strings); -} - -void piomemerror(iosystem_desc_t ios, size_t req, char *fname, const int line){ - char msg[80]; - sprintf(msg,"out of memory requesting: %ld",req); - cn_buffer_report(ios,false); - piodie(msg,fname,line); -} - - -void piodie(const char *msg,const char *fname, const int line){ - fprintf(stderr,"Abort with message %s in file %s at line %d\n",msg,fname,line); - - print_trace(stderr); -#ifdef MPI_SERIAL - abort(); -#else - MPI_Abort(MPI_COMM_WORLD, -1); -#endif -} - - -void pioassert(_Bool expression, const char *msg, const char *fname, const int line) -{ -#ifndef NDEBUG - if(! expression){ - piodie(msg,fname,line); - } -#endif - -} - -/** Check the result of a netCDF API call. - * - * @param file pointer to the PIO structure describing this file. - * @param status the return value from the netCDF call. - * @param fname the name of the code file. - * @param line the line number of the netCDF call in the code. - * - * @return the error code -*/ -int check_netcdf(file_desc_t *file, int status, const char *fname, const int line){ - iosystem_desc_t *ios; - int ierr; - char errstr[160]; - ios = file->iosystem; - ierr = PIO_NOERR; - - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->iomaster){ - if(status != NC_NOERR && (ios->error_handler == PIO_INTERNAL_ERROR)) - piodie(nc_strerror(status),fname,line); -// fprintf(stderr,"NETCDF ERROR: %s %s %d\n",nc_strerror(status),fname,line); - } - if(ios->error_handler == PIO_INTERNAL_ERROR){ - if(status != NC_NOERR) - MPI_Abort(MPI_COMM_WORLD,status); - // abort - }else if(ios->error_handler==PIO_BCAST_ERROR){ - ierr =MPI_Bcast(&status, 1, MPI_INTEGER, ios->ioroot, ios->my_comm); - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - if(status != NC_NOERR && (ios->error_handler == PIO_INTERNAL_ERROR)) { - // fprintf(stderr,"PNETCDF ERROR: %s %s %d\n",ncmpi_strerror(status),fname,line); - piodie(ncmpi_strerror(status),fname,line); - } - if(ios->error_handler==PIO_BCAST_ERROR){ - ierr = MPI_Bcast(&status, 1, MPI_INTEGER, ios->ioroot, ios->my_comm); - } - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - return status; -} - -int iotype_error(const int iotype, const char *fname, const int line) -{ - fprintf(stderr, "ERROR: iotype %d not defined in build %s %d\n", iotype, fname,line); - return(PIO_EBADIOTYPE); -} - -io_region *alloc_region(const int ndims) -{ - io_region *region; - - region = (io_region *) bget(sizeof(io_region)); - region->start = (PIO_Offset *) bget(ndims* sizeof(PIO_Offset)); - region->count = (PIO_Offset *) bget(ndims* sizeof(PIO_Offset)); - region->loffset = 0; - region->next=NULL; - for(int i=0;i< ndims; i++){ - region->start[i] = 0; - region->count[i] = 0; - } - return region; -} - -io_desc_t *malloc_iodesc(const int piotype, const int ndims) -{ - io_desc_t *iodesc; - iodesc = (io_desc_t *) bget(sizeof(io_desc_t)); - - if(iodesc == NULL) - fprintf(stderr,"ERROR: allocation error \n"); - - switch(piotype){ - case PIO_REAL: - iodesc->basetype=MPI_FLOAT; - break; - case PIO_DOUBLE: - iodesc->basetype=MPI_DOUBLE; - break; - case PIO_CHAR: - iodesc->basetype=MPI_CHAR; - break; - case PIO_INT: - default: - iodesc->basetype = MPI_INTEGER; - break; - } - iodesc->rearranger = 0; - iodesc->maxregions=1; - iodesc->rfrom = NULL; - iodesc->scount = NULL; - iodesc->rtype = NULL; - iodesc->stype = NULL; - iodesc->num_stypes = 0; - iodesc->sindex = NULL; - iodesc->rindex = NULL; - iodesc->rcount = NULL; - iodesc->ioid=-1; - iodesc->llen=0; - iodesc->maxiobuflen=0; - iodesc->holegridsize=0; - iodesc->maxbytes=0; - iodesc->ndims = ndims; - iodesc->firstregion = alloc_region(ndims); - iodesc->fillregion = NULL; - - iodesc->handshake=swapm_defaults.handshake; - iodesc->isend=swapm_defaults.isend; - iodesc->max_requests=swapm_defaults.nreqs; - - return iodesc; -} - -void free_region_list(io_region *top) -{ - io_region *ptr, *tptr; - - ptr = top; - while(ptr != NULL) { - if(ptr->start != NULL) - brel(ptr->start); - if(ptr->count != NULL) - brel(ptr->count); - tptr=ptr; - ptr=ptr->next; - brel(tptr); - } - -} - - -int PIOc_freedecomp(int iosysid, int ioid) -{ - iosystem_desc_t *ios; - io_desc_t *iodesc; - int i; - - ios = pio_get_iosystem_from_id(iosysid); - if(ios == NULL) - return PIO_EBADID; - - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL) - return PIO_EBADID; - - if(iodesc->rfrom != NULL) - brel(iodesc->rfrom); - if(iodesc->rtype != NULL){ - for(i=0; inrecvs; i++){ - if(iodesc->rtype[i] != MPI_DATATYPE_NULL){ - MPI_Type_free(iodesc->rtype+i); - } - } - brel(iodesc->rtype); - } - if(iodesc->stype != NULL){ - for(i=0; inum_stypes; i++){ - if(iodesc->stype[i] != MPI_DATATYPE_NULL){ - MPI_Type_free(iodesc->stype+i); - } - } - iodesc->num_stypes = 0; - brel(iodesc->stype); - } - if(iodesc->scount != NULL) - brel(iodesc->scount); - if(iodesc->rcount != NULL) - brel(iodesc->rcount); - if(iodesc->sindex != NULL) - brel(iodesc->sindex); - if(iodesc->rindex != NULL) - brel(iodesc->rindex); - - - if(iodesc->firstregion != NULL) - free_region_list(iodesc->firstregion); - - if(iodesc->rearranger == PIO_REARR_SUBSET){ - MPI_Comm_free(&(iodesc->subset_comm)); - } - - return pio_delete_iodesc_from_list(ioid); - - -} - -int PIOc_readmap(const char file[], int *ndims, int *gdims[], PIO_Offset *fmaplen, PIO_Offset *map[], const MPI_Comm comm) -{ - int npes, myrank; - int rnpes, rversno; - int j; - int *tdims; - PIO_Offset *tmap; - MPI_Status status; - PIO_Offset maplen; - - MPI_Comm_size(comm, &npes); - MPI_Comm_rank(comm, &myrank); - - if(myrank == 0) { - FILE *fp = fopen(file, "r"); - if(fp==NULL) - piodie("Failed to open dof file",__FILE__,__LINE__); - - fscanf(fp,"version %d npes %d ndims %d\n",&rversno, &rnpes,ndims); - - if(rversno != versno) - piodie("Attempt to read incompatable map file version",__FILE__,__LINE__); - - if(rnpes < 1 || rnpes > npes) - piodie("Incompatable pe count in map file ",__FILE__,__LINE__); - - MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm); - MPI_Bcast(ndims, 1, MPI_INT, 0, comm); - tdims = (int *) calloc((*ndims), sizeof(int)); - for(int i=0;i<(*ndims);i++) - fscanf(fp,"%d ",tdims+i); - - MPI_Bcast(tdims, (*ndims), MPI_INT, 0, comm); - - for(int i=0; i< rnpes; i++){ - fscanf(fp,"%d %ld",&j,&maplen); - if( j != i) // Not sure how this could be possible - piodie("Incomprehensable error reading map file ",__FILE__,__LINE__); - - tmap = (PIO_Offset *) malloc(maplen*sizeof(PIO_Offset)); - for(j=0;j0){ - MPI_Send(&maplen, 1, PIO_OFFSET, i, i+npes, comm); - MPI_Send(tmap, maplen, PIO_OFFSET, i, i, comm); - free(tmap); - }else{ - *map = tmap; - *fmaplen = maplen; - } - } - fclose(fp); - }else{ - MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm); - MPI_Bcast(ndims, 1, MPI_INT, 0, comm); - tdims = (int *) calloc((*ndims), sizeof(int)); - MPI_Bcast(tdims, (*ndims), MPI_INT, 0, comm); - - if(myrank -#include -void print_memusage() -{ - uint64_t shared, persist, heapavail, stackavail, stack, heap, guard, mmap; - - Kernel_GetMemorySize(KERNEL_MEMSIZE_SHARED, &shared); - Kernel_GetMemorySize(KERNEL_MEMSIZE_PERSIST, &persist); - Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPAVAIL, &heapavail); - Kernel_GetMemorySize(KERNEL_MEMSIZE_STACKAVAIL, &stackavail); - Kernel_GetMemorySize(KERNEL_MEMSIZE_STACK, &stack); - Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap); - Kernel_GetMemorySize(KERNEL_MEMSIZE_GUARD, &guard); - Kernel_GetMemorySize(KERNEL_MEMSIZE_MMAP, &mmap); - - printf("Allocated heap: %.2f MB, avail. heap: %.2f MB\n", (double)heap/(1024*1024), (double)heapavail/(1024*1024)); - printf("Allocated stack: %.2f MB, avail. stack: %.2f MB\n", (double)stack/(1024*1024), (double)stackavail/(1024*1024)); - printf("Memory: shared: %.2f MB, persist: %.2f MB, guard: %.2f MB, mmap: %.2f MB\n", (double)shared/(1024*1024), (double)persist/(1024*1024), (double)guard/(1024*1024), (double)mmap/(1024*1024)); - - return; -} -#endif -*/ diff --git a/externals/pio2/src/clib/topology.c b/externals/pio2/src/clib/topology.c deleted file mode 100644 index 9334d127c17..00000000000 --- a/externals/pio2/src/clib/topology.c +++ /dev/null @@ -1,363 +0,0 @@ -#if defined(BGQDONT) - -#include -#include - -#include - -#include -#include -#include -#include - -#define Personality Personality_t - -int rank; -int np; -int my_name_len; -char my_name[255]; - -void identity(const MPI_Comm comm, int *iotask) -{ - MPIX_Hardware_t hw; - char message[100]; - /* Number of MPI tasks per Pset */ - int coreId; - int *TasksPerPset; - int *tmp; - int i,ierr; - Personality_t pers; - - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&np); - MPI_Get_processor_name(my_name, &my_name_len); - - - MPIX_Hardware(&hw); - - Kernel_GetPersonality(&pers, sizeof(pers)); - - int numIONodes,numPsets,numNodesInPset,rankInPset; - int numpsets, psetID, psetsize, psetrank; - - bgq_pset_info (comm, &numpsets, &psetID, &psetsize, &psetrank); - - numIONodes = numpsets; - numNodesInPset = psetsize; - rankInPset = rank; - numPsets = numpsets; - - if(rank == 0) { printf("number of IO nodes in block: %i \n",numIONodes);} - if(rank == 0) { printf("number of Psets in block : %i \n",numPsets);} - if(rank == 0) { printf("number of compute nodes in Pset: %i \n",numNodesInPset);} - - int psetNum; - psetNum = psetID; - -#ifdef DEBUG - if((*iotask)>0) { - printf( "%04i (%-50s %s) %i yes\n", rank, my_name, message, psetNum ); - } else { - printf( "%04i (%-50s %s) %i --\n", rank, my_name, message, psetNum); - } - printf("MPI task %6i is rank %3i in Pset: %3i \n",rank, rankInPset,psetNum); -#endif - /* Determine which core on node.... I don't want to put more than one io-task per node */ - coreId = Kernel_PhysicalProcessorID (); - - TasksPerPset = malloc(numPsets*sizeof(int)); - tmp = malloc(numPsets*sizeof(int)); - for(i=0;i 0) { - - - Kernel_GetPersonality(&pers, sizeof(pers)); - - int numIONodes,numPsets,numNodesInPset,rankInPset; - int numiotasks_per_node,remainder,numIONodes_per_pset; - int lstride; - - /* Number of computational nodes in processor set */ - - int numpsets, psetID, psetsize, psetrank; - bgq_pset_info (comm,&numpsets, &psetID, &psetsize, &psetrank); - numIONodes = numpsets; - numNodesInPset = psetsize; - - /* printf("Determine io tasks: me %i : nodes in pset= %i ionodes = %i\n", rank, numNodesInPset, numIONodes); */ - - if((*numiotasks) < 0 ) { - /* negative numiotasks value indicates that this is the number per IO-node */ - (*numiotasks) = - (*numiotasks); - if((*numiotasks) > numNodesInPset) { - numiotasks_per_node = numNodesInPset; - } else { - numiotasks_per_node = (*numiotasks); - } - remainder = 0; - } else if ((*numiotasks) > 0 ) { - /* balance the number of iotasks to number of IO nodes */ - numiotasks_per_node = floor((float)(*numiotasks)/ (float) numIONodes); - /* put a minumum here so that we have a chance - though this may be too low */ - if (numiotasks_per_node < 1) { - numiotasks_per_node = 1; - *numiotasks = numIONodes; - } - remainder = (*numiotasks) - numiotasks_per_node * numIONodes; - } else if ((*numiotasks) == 0 ) { - if((*stride) > 0) { - numiotasks_per_node = numNodesInPset/(*stride); - if (numiotasks_per_node < 1) { - numiotasks_per_node = 1; - *numiotasks = numIONodes; - } - } else { - numiotasks_per_node = 8; /* default number of IO-client per IO-node is not otherwise specificied */ - } - remainder = 0; - } - - /* number of IO nodes with a larger number of io-client per io-node */ - if(remainder > 0) { - if(rank ==0) {printf("Unbalanced IO-configuration: %i IO-nodes have %i IO-clients : %i IO-nodes have %i IO-clients \n", - remainder, numiotasks_per_node+1, numIONodes-remainder,numiotasks_per_node);} - lstride = min(np,floor((float)numNodesInPset/(float)(numiotasks_per_node+1))); - } else { - if(rank == 0) { - printf("Balanced IO-configuration: %i IO-nodes have %i IO-clients\n",numIONodes-remainder, numiotasks_per_node); - } - lstride = min(np,floor((float)numNodesInPset/(float)numiotasks_per_node)); - } - - /* Number of processor sets */ - numPsets = numpsets; - - /* number of IO nodes in processor set (I need to add - code to deal with the case where numIONodes_per_pset != 1 works - correctly) */ - numIONodes_per_pset = numIONodes/numPsets; - - /* Determine which core on node.... I don't want to put more than one io-task per node */ - coreId = Kernel_PhysicalProcessorID (); - - /* What is the rank of this node in the processor set */ - - psetNum = psetID; - rankInPset = psetrank; - - /* printf("Pset #: %i has %i nodes in Pset; base = %i\n",psetNum,numNodesInPset, *base); */ - - (*iamIOtask) = false; /* initialize to zero */ - - if (numiotasks_per_node == numNodesInPset)(*base) = 0; /* Reset the base to 0 if we are using all tasks */ - - - /* start stridding MPI tasks from base task */ - iam = max(0,rankInPset-(*base)); - if (iam >= 0) { - /* mark tasks that will be IO-tasks or IO-clients */ - /* printf("iam = %d lstride = %d coreID = %d\n",iam,lstride,coreId); */ - if((iam % lstride == 0) && (coreId == 0) ) { /* only io tasks indicated by stride and coreId = 0 */ - if((iam/lstride) < numiotasks_per_node) { - /* only set the first (numiotasks_per_node - 1) tasks */ - (*iamIOtask) = true; - } else if ((iam/lstride) == numiotasks_per_node) { - /* If there is an uneven number of io-clients to io-nodes - allocate the first remainder - 1 processor sets to - have a total of numiotasks_per_node */ - if(psetNum < remainder) {(*iamIOtask) = true; - }; - } - } - /* printf("comm = %d iam = %d lstride = %d coreID = %d iamIOtask = %i \n",comm, iam,lstride,coreId,(*iamIOtask)); */ - } - }else{ - /* We are not doing rearrangement.... so all tasks are io-tasks */ - (*iamIOtask) = true; - } - - /*printf("comm = %d myrank = %i iotask = %i \n", comm, rank, (*iamIOtask));*/ - - /* now we need to correctly determine the numiotasks */ - MPI_Allreduce(iamIOtask, &task_count, 1, MPI_INT, MPI_SUM, comm); - - (*numiotasks) = task_count; - - -} - -int bgq_ion_id (void) -{ - int iA, iB, iC, iD, iE; /* The local node's coordinates */ - int nA, nB, nC, nD, nE; /* Size of each torus dimension */ - int brA, brB, brC, brD, brE; /* The bridge node's coordinates */ - int io_node_route_id; - - Personality_t personality; - - Kernel_GetPersonality(&personality, sizeof(personality)); - - iA = personality.Network_Config.Acoord; - iB = personality.Network_Config.Bcoord; - iC = personality.Network_Config.Ccoord; - iD = personality.Network_Config.Dcoord; - iE = personality.Network_Config.Ecoord; - - nA = personality.Network_Config.Anodes; - nB = personality.Network_Config.Bnodes; - nC = personality.Network_Config.Cnodes; - nD = personality.Network_Config.Dnodes; - nE = personality.Network_Config.Enodes; - - brA = personality.Network_Config.cnBridge_A; - brB = personality.Network_Config.cnBridge_B; - brC = personality.Network_Config.cnBridge_C; - brD = personality.Network_Config.cnBridge_D; - brE = personality.Network_Config.cnBridge_E; - -/* -* This is the bridge node, numbered in ABCDE order, E increments first. -* It is considered the unique "io node route identifer" because each -* bridge node only has one torus link to one io node. -*/ - - io_node_route_id = brE + brD*nE + brC*nD*nE + brB*nC*nD*nE + brA*nB*nC*nD*nE; - - return io_node_route_id; - -} - - - -int bgq_pset_info (MPI_Comm comm, int* tot_pset, int* psetID, int* pset_size, int* rank_in_pset) -{ - MPI_Comm comp_comm, pset_comm, bridge_comm; - int comp_rank, status, key, bridge_root, tot_bridges, cur_pset, itr, t_buf; - int temp_id, rem_psets; - MPI_Status mpi_status; - - MPI_Comm_rank (comm, &comp_rank); - - status = MPI_Comm_dup ( comm, &comp_comm); - if ( MPI_SUCCESS != status) - { - printf(" Error duplicating communicator \n"); - MPI_Abort(comm, status); - } - - // Compute the ION BridgeNode ID - key = bgq_ion_id (); - - // Create the pset_comm per bridge node - status = MPI_Comm_split ( comp_comm, key, comp_rank, &pset_comm); - if ( MPI_SUCCESS != status) - { - printf(" Error splitting communicator \n"); - MPI_Abort(comm, status); - } - - // Calculate the rank in pset and pset size - MPI_Comm_rank (pset_comm, rank_in_pset); - MPI_Comm_size (pset_comm, pset_size); - - // Create the Bridge root nodes communicator - bridge_root = 0; - if (0 == *rank_in_pset) - bridge_root = 1; - - // Calculate the total number of bridge nodes / psets - tot_bridges = 0; - MPI_Allreduce (&bridge_root, &tot_bridges, 1, MPI_INT, MPI_SUM, comm); - - *tot_pset = tot_bridges; - - // Calculate the Pset ID - cur_pset = 0; - rem_psets = tot_bridges; - if ((0 == comp_rank) && (bridge_root ==1)) - { - *psetID = 0; - rem_psets = tot_bridges-1; - cur_pset++; - } - - t_buf = 0; // Dummy value - if (0 == comp_rank) - { - for (itr = 0; itr < rem_psets; itr++) - { - MPI_Recv (&t_buf,1, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG,comm, &mpi_status); - MPI_Send (&cur_pset, 1, MPI_INT, mpi_status.MPI_SOURCE, 0, comm); - cur_pset++; - } - } - - if ((1 == bridge_root) && ( 0 != comp_rank)) - { - MPI_Send (&t_buf, 1, MPI_INT, 0, 0, comm); - MPI_Recv (&temp_id,1, MPI_INT, 0, 0, comm, &mpi_status); - - *psetID = temp_id; - /*printf (" Pset ID is %d \n", *psetID);*/ - } - // Broadcast the PSET ID to all ranks in the psetcomm - MPI_Bcast ( psetID, 1, MPI_INT, 0, pset_comm); - - // Free the split comm - MPI_Comm_free (&pset_comm); - - MPI_Barrier (comm); - - return 0; -} - -#endif diff --git a/externals/pio2/src/flib/CMakeLists.txt b/externals/pio2/src/flib/CMakeLists.txt deleted file mode 100644 index cef209ba8b9..00000000000 --- a/externals/pio2/src/flib/CMakeLists.txt +++ /dev/null @@ -1,269 +0,0 @@ -cmake_minimum_required (VERSION 2.8.12) -project (PIOF Fortran) -include (CheckFunctionExists) -include (ExternalProject) -include (LibCheck) - -#============================================================================== -# DEFINE THE TARGET -#============================================================================== - -set (PIO_Fortran_SRCS pio_nf.F90 - pio.F90 - pio_kinds.F90 - pio_types.F90 - piolib_mod.F90 - pio_support.F90) - -set (PIO_GenF90_SRCS pionfatt_mod.F90 - pionfput_mod.F90 - pionfget_mod.F90 - piodarray.F90) - -set (PIO_Fortran_MODS ${CMAKE_CURRENT_BINARY_DIR}/pio.mod - ${CMAKE_CURRENT_BINARY_DIR}/pio_nf.mod - ${CMAKE_CURRENT_BINARY_DIR}/pio_types.mod - ${CMAKE_CURRENT_BINARY_DIR}/piolib_mod.mod - ${CMAKE_CURRENT_BINARY_DIR}/pionfget_mod.mod - ${CMAKE_CURRENT_BINARY_DIR}/pio_kinds.mod - ${CMAKE_CURRENT_BINARY_DIR}/pio_support.mod - ${CMAKE_CURRENT_BINARY_DIR}/piodarray.mod - ${CMAKE_CURRENT_BINARY_DIR}/pionfatt_mod.mod - ${CMAKE_CURRENT_BINARY_DIR}/pionfput_mod.mod) - -add_library (piof ${PIO_Fortran_SRCS} ${PIO_GenF90_SRCS}) -if (NOT PIO_ENABLE_FORTRAN) - set_target_properties(piof PROPERTIES EXCLUDE_FROM_ALL TRUE) -endif () - -# Include flib source and binary directories (for Fortran modules) -target_include_directories (piof - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) - -# System and compiler CPP directives -target_compile_definitions (piof - PUBLIC ${CMAKE_SYSTEM_DIRECTIVE}) -target_compile_definitions (piof - PUBLIC ${CMAKE_Fortran_COMPILER_DIRECTIVE}) - -# Compiler-specific compile options -if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") - target_compile_options (piof - PRIVATE -ffree-line-length-none) -elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") - set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) -# target_compile_options (piof -# PRIVATE -mismatch_all) -endif() - -# Look for c_sizeof capability -check_macro (Fortran_CSIZEOF - NAME TryCSizeOf.f90 - HINTS ${CMAKE_MODULE_PATH} - COMMENT "whether Fortran compiler supports c_sizeof") -if (NOT Fortran_CSIZEOF) - target_compile_definitions (piof - PUBLIC NO_C_SIZEOF) -endif() - -# Look for filesystem hints -if (PIO_FILESYSTEM_HINTS) - if (PIO_FILESYSTEM_HINTS STREQUAL lustre) - message (STATUS "PIO will use lustre filesystem hints") - target_compile_definitions (piof - PUBLIC PIO_LUSTRE_HINTS) - elseif (PIO_FILESYSTEM_HINTS STREQUAL gpfs) - message (STATUS "PIO will use gpfs filesystem hints") - target_compile_definitions (piof - PUBLIC PIO_GPFS_HINTS) - else () - message (WARNING "${PIO_FILESYSTEM_HINTS} not valid option for PIO_FILESYSTEM_HINTS; use gpfs or lustre.") - endif () -endif () - -#============================================================================== -# DEFINE THE INSTALL -#============================================================================== - -# Library -install (TARGETS piof DESTINATION lib) - -# Fortran Modules -install (FILES ${PIO_Fortran_MODS} DESTINATION include) - -#============================================================================== -# DEFINE THE DEPENDENCIES -#============================================================================== - -#===== pioc ===== -target_link_libraries(piof - PUBLIC pioc) - -#===== genf90 ===== -if (DEFINED GENF90_PATH) - add_custom_target(genf90 - DEPENDS ${GENF90_PATH}/genf90.pl) -else () - ExternalProject_Add (genf90 - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/genf90 - GIT_REPOSITORY https://github.com/PARALLELIO/genf90 - GIT_TAG genf90_140121 - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "") - ExternalProject_Get_Property (genf90 SOURCE_DIR) - set (GENF90_PATH ${SOURCE_DIR}) - unset (SOURCE_DIR) -endif () -add_dependencies (piof genf90) - -#===== Fortran Source Generation with GenF90 ===== -foreach (SRC_FILE IN LISTS PIO_GenF90_SRCS) - add_custom_command (OUTPUT ${SRC_FILE} - COMMAND ${GENF90_PATH}/genf90.pl - ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in > ${SRC_FILE} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in genf90) -endforeach () - -#===== MPI ===== -if (PIO_USE_MPISERIAL) - find_package (MPISERIAL COMPONENTS Fortran REQUIRED) - if (MPISERIAL_Fortran_FOUND) - target_compile_definitions (piof - PRIVATE _MPISERIAL) - target_include_directories (piof - PUBLIC ${MPISERIAL_Fortran_INCLUDE_DIRS}) - target_link_libraries (piof - PUBLIC ${MPISERIAL_Fortran_LIBRARIES}) - - set (WITH_PNETCDF FALSE) - set (MPI_Fortran_INCLUDE_PATH ${MPISERIAL_Fortran_INCLUDE_DIRS}) - endif () -else () - find_package (MPI REQUIRED) -endif () - -# Check MPI I/O capabilities -find_path(MPIF_H_PATH - NAMES mpif.h - PATHS ${MPI_Fortran_INCLUDE_PATH} - NO_DEFAULT_PATH) -if (MPIF_H_PATH) - check_macro (MPI_HAS_MPIIO - NAME TryMPIIO.f90 - HINTS ${CMAKE_MODULE_PATH} - DEFINITIONS -I{MPIF_H_PATH} - COMMENT "whether MPIIO is supported") - if (${MPI_HAS_MPIIO}) - message (STATUS "MPIIO verified and enabled.") - target_compile_definitions (piof - PUBLIC USEMPIIO) - else () - message (STATUS "MPIIO failed verification and therefore disabled.") - endif () -else () - message (STATUS "MPIIO not detected and therefore disabled.") -endif () - -# Check for MPI Fortran module -find_path(MPIMOD_PATH - NAMES mpi.mod MPI.mod - HINTS ${MPI_Fortran_INCLUDE_PATH}) -if (PIO_ENABLE_TIMING) - find_package (GPTL COMPONENTS Fortran_Perf QUIET) -endif () - -check_macro (MPI_HAS_Fortran_MOD - NAME TryMPIMod.f90 - HINTS ${CMAKE_MODULE_PATH} - DEFINITIONS -I${MPIMOD_PATH} - COMMENT "whether MPI Fortran module is supported") -if (${MPI_HAS_Fortran_MOD}) - message (STATUS "MPI Fortran module verified and enabled.") -else () - message (STATUS "MPI Fortran module failed verification and therefore disabled.") - if (PIO_ENABLE_TIMING AND NOT GPTL_Fortran_Perf_FOUND) - target_compile_definitions (gptl - PUBLIC NO_MPIMOD) - endif() -endif () - -#===== GPTL ===== -if (PIO_ENABLE_TIMING) - if (GPTL_Fortran_Perf_FOUND) - message (STATUS "Found GPTL Fortran Perf: ${GPTL_Fortran_Perf_LIBRARIES}") - target_include_directories (piof - PUBLIC ${GPTL_Fortran_INCLUDE_DIRS}) - target_link_libraries (piof - PUBLIC ${GPTL_Fortran_LIBRARIES}) - else () - message (STATUS "Using internal GPTL Fortran library for timing") - target_link_libraries (piof - PUBLIC gptl) - endif () -endif () - -#===== NetCDF-Fortran ===== -find_package (NetCDF "4.3.3" COMPONENTS Fortran) -if (NetCDF_Fortran_FOUND) - target_include_directories (piof - PUBLIC ${NetCDF_Fortran_INCLUDE_DIRS}) - target_compile_definitions (piof - PUBLIC _NETCDF) - target_link_libraries (piof - PUBLIC ${NetCDF_Fortran_LIBRARIES}) - if (EXISTS ${NetCDF_Fortran_INCLUDE_DIR}/netcdf_par.h) - target_compile_definitions (piof - PUBLIC _NETCDF4) - endif () -else () - target_compile_definitions (piof - PUBLIC _NONETCDF) -endif () - -#===== PnetCDF ===== -if (WITH_PNETCDF) - find_package (PnetCDF "1.6" COMPONENTS Fortran REQUIRED) -endif () -if (PnetCDF_Fortran_FOUND) - target_include_directories (piof - PUBLIC ${PnetCDF_Fortran_INCLUDE_DIRS}) - target_compile_definitions (piof - PUBLIC _PNETCDF) - target_link_libraries (piof - PUBLIC ${PnetCDF_Fortran_LIBRARIES}) - - # Check library for varn functions - set (CMAKE_REQUIRED_LIBRARIES ${PnetCDF_Fortran_LIBRARY}) - check_function_exists (ncmpi_get_varn PnetCDF_Fortran_HAS_VARN) - if (PnetCDF_Fortran_HAS_VARN) - target_compile_definitions(piof - PUBLIC USE_PNETCDF_VARN - PUBLIC USE_PNETCDF_VARN_ON_READ) - endif() -else () - target_compile_definitions (piof - PUBLIC _NOPNETCDF) -endif () - -#===== Add EXTRAs ===== -target_include_directories (piof - PUBLIC ${PIO_Fortran_EXTRA_INCLUDE_DIRS}) -target_link_libraries (piof - PUBLIC ${PIO_Fortran_EXTRA_LIBRARIES}) -target_compile_options (piof - PRIVATE ${PIO_Fortran_EXTRA_COMPILE_OPTIONS}) -target_compile_definitions (piof - PUBLIC ${PIO_Fortran_EXTRA_COMPILE_DEFINITIONS}) -if (PIO_Fortran_EXTRA_LINK_FLAGS) - set_target_properties(piof PROPERTIES - LINK_FLAGS ${PIO_Fortran_EXTRA_LINK_FLAGS}) -endif () - -#===== Check for necessities ===== -if (NOT PnetCDF_Fortran_FOUND AND NOT NetCDF_Fortran_FOUND) - message (FATAL_ERROR "Must have PnetCDF and/or NetCDF Fortran libraries") -endif () - diff --git a/externals/pio2/src/gptl/CMakeLists.txt b/externals/pio2/src/gptl/CMakeLists.txt deleted file mode 100644 index 02301f58183..00000000000 --- a/externals/pio2/src/gptl/CMakeLists.txt +++ /dev/null @@ -1,156 +0,0 @@ -cmake_minimum_required (VERSION 2.8.12) -project (GPTL C Fortran) -include (CheckFunctionExists) -include (FortranCInterface) -FortranCInterface_HEADER(cmake_fortran_c_interface.h - MACRO_NAMESPACE "FCI_") -include (LibCheck) - -#============================================================================== -# DEFINE THE TARGET -#============================================================================== - -set (GPTL_C_SRCS GPTLget_memusage.c - GPTLprint_memusage.c - GPTLutil.c - f_wrappers.c - gptl.c - gptl_papi.c - threadutil.c) - -set (GPTL_Fortran_SRCS perf_mod.F90 - perf_utils.F90) - -set (GPTL_Fortran_MODS ${CMAKE_CURRENT_BINARY_DIR}/perf_mod.mod - ${CMAKE_CURRENT_BINARY_DIR}/perf_utils.mod) - -add_library (gptl ${GPTL_Fortran_SRCS} ${GPTL_C_SRCS}) - -target_include_directories (gptl - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) - -target_compile_definitions (gptl - PUBLIC INCLUDE_CMAKE_FCI) - -if (CMAKE_SYSTEM_NAME MATCHES "AIX") - target_compile_definitions (gptl - PUBLIC _AIX) -else () - target_compile_definitions (gptl - PUBLIC ${CMAKE_SYSTEM_DIRECTIVE}) -endif () - -target_compile_definitions (gptl - PUBLIC ${CMAKE_Fortran_COMPILER_DIRECTIVE}) - -if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") - set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) -# target_compile_options (gptl -# PRIVATE -mismatch_all) -endif () - -#============================================================================== -# DEFINE THE INSTALL -#============================================================================== - -# Library -install (TARGETS gptl DESTINATION lib) - -# Header/Include File -install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/gptl.h DESTINATION include) - -# Fortran Modules -install (FILES ${GPTL_Fortran_MODS} DESTINATION include) - -#============================================================================== -# DEFINE THE DEPENDENCIES -#============================================================================== - -#===== PAPI ===== -if (ENABLE_PAPI) - find_package (PAPI) - if (PAPI_FOUND) - target_compile_definitions (gptl - PUBLIC HAVE_PAPI) - target_include_directories (gptl - PUBLIC ${PAPI_INCLUDE_DIRECTORIES}) - target_link_libraries (gptl - PUBLIC ${PAPI_LIBRARIES}) - endif () -endif () - -#===== LIBRT ===== -if (ENABLE_LIBRT) - find_package (LIBRT) - if (LIBRT_FOUND) - target_compile_definitions (gptl - PUBLIC HAVE_LIBRT) - target_include_directories (gptl - PUBLIC ${LIBRT_INCLUDE_DIRECTORIES}) - target_link_libraries (gptl - PUBLIC ${LIBRT_LIBRARIES}) - endif () -endif () - -#===== MPI ===== -if (PIO_USE_MPISERIAL) - find_package (MPISERIAL COMPONENTS C Fortran REQUIRED) - if (MPISERIAL_C_FOUND AND MPISERIAL_Fortran_FOUND) - target_compile_definitions (gptl - PRIVATE HAVE_MPI) - target_include_directories (gptl - PUBLIC ${MPISERIAL_C_INCLUDE_DIRS} - PUBLIC ${MPISERIAL_Fortran_INCLUDE_DIRS}) - target_link_libraries (gptl - PUBLIC ${MPISERIAL_C_LIBRARIES} - PUBLIC ${MPISERIAL_Fortran_LIBRARIES}) - - set (MPI_C_LIBRARIES ${MPISERIAL_C_LIBRARIES}) - set (MPI_Fortran_INCLUDE_PATH ${MPISERIAL_Fortran_INCLUDE_DIRS}) - endif () -else () - find_package (MPI REQUIRED) - if (MPI_C_FOUND AND MPI_Fortran_FOUND) - target_compile_definitions (gptl - PUBLIC HAVE_MPI) - endif () -endif () - -# Check MPI library for Comm_f2c function -set (CMAKE_REQUIRED_LIBRARIES ${MPI_C_LIBRARIES}) -check_function_exists (MPI_Comm_f2c MPI_HAS_COMM_F2C) -if (NOT MPI_HAS_COMM_F2C) - target_compile_definitions(gptl - PRIVATE NO_COMM_F2C) -endif () - -# Check for MPI Fortran module -find_path(MPIMOD_PATH - NAMES mpi.mod MPI.mod - PATHS ${MPI_Fortran_INCLUDE_PATH}) -check_macro (MPI_HAS_Fortran_MOD - NAME TryMPIMod.f90 - HINTS ${CMAKE_MODULE_PATH} - DEFINITIONS -I${MPIMOD_PATH} - COMMENT "whether MPI Fortran module is supported") -if (${MPI_HAS_Fortran_MOD}) - message (STATUS "MPI Fortran module verified and enabled.") -else () - message (STATUS "MPI Fortran module failed verification and therefore disabled.") - target_compile_definitions (gptl - PUBLIC NO_MPIMOD) -endif () - -#===== GetTimeOfDay ===== -if (NOT DEFINED SYSTEM_HAS_GETTIMEOFDAY) - get_target_property (GPTL_LINK_LIBRARIES gptl LINK_LIBRARIES) - set (CMAKE_REQUIRED_LIBRARIES ${GPTL_LINK_LIBRARIES}) - check_function_exists (gettimeofday GETTIMEOFDAY) - if (NOT GETTIMEOFDAY) - target_compile_definitions(gptl - PUBLIC NO_GETTIMEOFDAY) - endif () - set (SYSTEM_HAS_GETTIMEOFDAY ${GETTIMEOFDAY} - CACHE INTERNAL "Whether the gettimeofday function could be found") -endif () diff --git a/externals/pio2/src/gptl/COPYING b/externals/pio2/src/gptl/COPYING deleted file mode 100644 index 94a9ed024d3..00000000000 --- a/externals/pio2/src/gptl/COPYING +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/externals/pio2/tests/general/CMakeLists.txt b/externals/pio2/tests/general/CMakeLists.txt deleted file mode 100644 index 448918a1a77..00000000000 --- a/externals/pio2/tests/general/CMakeLists.txt +++ /dev/null @@ -1,327 +0,0 @@ -include (LibMPI) - -#============================================================================== -# GENERATE TARGET SOURCES -#============================================================================== - -SET(GENERATED_SRCS pio_file_simple_tests.F90 - pio_init_finalize.F90 - pio_fail.F90 - pio_file_fail.F90 - ncdf_simple_tests.F90 - ncdf_get_put.F90 - ncdf_fail.F90 - pio_decomp_tests.F90 - pio_decomp_fillval.F90) - -foreach (SRC_FILE IN LISTS GENERATED_SRCS) - add_custom_command ( - OUTPUT ${SRC_FILE} - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tf_f90gen.pl - --annotate-source - --out=${CMAKE_CURRENT_BINARY_DIR}/${SRC_FILE} - ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in) -endforeach () - -if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") - add_definitions(-ffree-line-length-none) -endif() - -if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") - set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) -# target_compile_options (gptl -# PRIVATE -mismatch_all) -endif () - -#============================================================================== -# DEFINE THE TARGETS AND TESTS -#============================================================================== - -# Test Timeout (4 min = 240 sec) -set (DEFAULT_TEST_TIMEOUT 240) - -#===== pio_init_finalize ===== -add_executable (pio_init_finalize EXCLUDE_FROM_ALL - pio_init_finalize.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (pio_init_finalize piof) -add_dependencies (tests pio_init_finalize) - -if (PIO_USE_MPISERIAL) - add_test(NAME init_finialize_1_proc - COMMAND pio_init_finalize) - set_tests_properties(init_finialize_1_proc - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_test(NAME init_finialize_2_proc - COMMAND pio_init_finalize) - set_tests_properties(init_finialize_2_proc - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_test(NAME init_finalize_2_proc_with_args - COMMAND pio_init_finalize --pio-tf-stride=2 --pio-tf-num-aggregators=2) - set_tests_properties(init_finalize_2_proc_with_args - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(init_finialize_1_proc - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize - NUMPROCS 1 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_mpi_test(init_finialize_2_proc - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_mpi_test(init_finalize_2_proc_with_args - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize - ARGUMENTS --pio-tf-stride=2 --pio-tf-num-aggregators=2 - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== pio_file_simple_tests ===== -add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL - pio_file_simple_tests.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (pio_file_simple_tests piof) -add_dependencies (tests pio_file_simple_tests) - -if (PIO_USE_MPISERIAL) - add_test(NAME pio_file_simple_tests - COMMAND pio_file_simple_tests) - set_tests_properties(pio_file_simple_tests - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(pio_file_simple_tests - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== pio_file_fail ===== -add_executable (pio_file_fail EXCLUDE_FROM_ALL - pio_file_fail.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (pio_file_fail piof) -if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") - target_compile_options (pio_init_finalize - PRIVATE -ffree-line-length-none) -endif() -add_dependencies (tests pio_file_fail) - -if (PIO_USE_MPISERIAL) - add_test(NAME pio_file_fail - COMMAND pio_file_fail) - set_tests_properties(pio_file_fail - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(pio_file_fail - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_fail - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== ncdf_simple_tests ===== -add_executable (ncdf_simple_tests EXCLUDE_FROM_ALL - ncdf_simple_tests.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (ncdf_simple_tests piof) -add_dependencies (tests ncdf_simple_tests) - -if (PIO_USE_MPISERIAL) - add_test(NAME ncdf_simple_tests - COMMAND ncdf_simple_tests) - set_tests_properties(ncdf_simple_tests - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(ncdf_simple_tests - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_simple_tests - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== ncdf_get_put ===== -add_executable (ncdf_get_put EXCLUDE_FROM_ALL - ncdf_get_put.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (ncdf_get_put piof) -add_dependencies (tests ncdf_get_put) - -if (PIO_USE_MPISERIAL) - add_test(NAME ncdf_get_put_1proc - COMMAND ncdf_get_put) - set_tests_properties(ncdf_get_put_1proc - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME ncdf_get_put_2proc - COMMAND ncdf_get_put) - set_tests_properties(ncdf_get_put_2proc - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(ncdf_get_put_1proc - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_get_put - NUMPROCS 1 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(ncdf_get_put_2proc - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_get_put - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== ncdf_fail ===== -add_executable (ncdf_fail EXCLUDE_FROM_ALL - ncdf_fail.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (ncdf_fail piof) -add_dependencies (tests ncdf_fail) - -if (PIO_USE_MPISERIAL) - add_test(NAME ncdf_fail - COMMAND ncdf_fail) - set_tests_properties(ncdf_fail - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(ncdf_fail - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_fail - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== pio_decomp_tests ===== -add_executable (pio_decomp_tests EXCLUDE_FROM_ALL - pio_decomp_tests.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (pio_decomp_tests piof) -add_dependencies (tests pio_decomp_tests) - -if (PIO_USE_MPISERIAL) - add_test(NAME pio_decomp_tests_1p - COMMAND pio_decomp_tests) - set_tests_properties(pio_decomp_tests_1p - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_2p - COMMAND pio_decomp_tests) - set_tests_properties(pio_decomp_tests_2p - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_3p - COMMAND pio_decomp_tests) - set_tests_properties(pio_decomp_tests_3p - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_test(NAME pio_decomp_tests_4p_1agg - COMMAND pio_decomp_tests --pio-tf-num-aggregators=1) - set_tests_properties(pio_decomp_tests_4p_1agg - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_4p_2agg - COMMAND pio_decomp_tests --pio-tf-num-aggregators=2) - set_tests_properties(pio_decomp_tests_4p_2agg - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_4p_3agg - COMMAND pio_decomp_tests --pio-tf-num-aggregators=3) - set_tests_properties(pio_decomp_tests_4p_3agg - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_test(NAME pio_decomp_tests_4p_1iop - COMMAND pio_decomp_tests --pio-tf-num-io-tasks=1) - set_tests_properties(pio_decomp_tests_4p_1iop - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_4p_2iop - COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2) - set_tests_properties(pio_decomp_tests_4p_2iop - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_4p_3iop - COMMAND pio_decomp_tests --pio-tf-num-io-tasks=3) - set_tests_properties(pio_decomp_tests_4p_3iop - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_test(NAME pio_decomp_tests_4p_2iop_2str - COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2 --pio-tf-stride=2) - set_tests_properties(pio_decomp_tests_4p_2iop_2str - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_test(NAME pio_decomp_tests_4p_2iop_1agg - COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1) - set_tests_properties(pio_decomp_tests_4p_2iop_1agg - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(pio_decomp_tests_1p - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - NUMPROCS 1 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_2p - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_3p - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - NUMPROCS 3 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_mpi_test(pio_decomp_tests_4p_1agg - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-aggregators=1 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_4p_2agg - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-aggregators=2 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_4p_3agg - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-aggregators=3 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_mpi_test(pio_decomp_tests_4p_1iop - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-io-tasks=1 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_4p_2iop - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-io-tasks=2 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_4p_3iop - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-io-tasks=3 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - - add_mpi_test(pio_decomp_tests_4p_2iop_2str - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_decomp_tests_4p_2iop_1agg - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests - ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -#===== pio_decomp_fillval ===== -add_executable (pio_decomp_fillval EXCLUDE_FROM_ALL - pio_decomp_fillval.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (pio_decomp_fillval piof) -add_dependencies (tests pio_decomp_fillval) - -add_executable (pio_decomp_fillval2 EXCLUDE_FROM_ALL - pio_decomp_fillval2.F90 - ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) -target_link_libraries (pio_decomp_fillval2 piof) -add_dependencies (tests pio_decomp_fillval2) - -if (PIO_USE_MPISERIAL) - add_test(NAME pio_decomp_fillval2 - COMMAND pio_decomp_fillval2) - set_tests_properties(pio_decomp_fillval2 - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(pio_decomp_fillval2 - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_fillval2 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () diff --git a/externals/pio2/tests/general/ncdf_get_put.F90.in b/externals/pio2/tests/general/ncdf_get_put.F90.in deleted file mode 100644 index 5f73ead1e02..00000000000 --- a/externals/pio2/tests/general/ncdf_get_put.F90.in +++ /dev/null @@ -1,120 +0,0 @@ -PIO_TF_TEMPLATE -PIO_TF_AUTO_TEST_SUB_BEGIN test_put_1dval - Implicit none - type(file_desc_t) :: pio_file - character(len=PIO_TF_MAX_STR_LEN) :: filename - type(var_desc_t) :: pio_var, pio_cvar - integer :: pio_dim - integer, parameter :: DIM_LEN = 100 - PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: val - CHARACTER(len=DIM_LEN) :: cval - integer, dimension(:), allocatable :: iotypes - character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs - integer :: num_iotypes - integer :: i, ret - - val = pio_tf_world_sz_ - cval = "DUMMY_STRING" - num_iotypes = 0 - call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) - filename = "test_pio_ncdf_get_put.testfile" - do i=1,num_iotypes - PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) - ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) - PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) - - ! Since file is just created no need to enter redef - ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim) - PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) - - ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) - PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) - - ret = PIO_def_var(pio_file, 'dummy_var_put_cval', PIO_char, (/pio_dim/), pio_cvar) - PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename)) - - ret = PIO_put_att(pio_file, pio_var, 'dummy_att_put_val', val); - PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(filename)) - - ret = PIO_put_att(pio_file, pio_cvar, 'dummy_att_put_cval', cval); - PIO_TF_CHECK_ERR(ret, "Failed to put char attribute:" // trim(filename)) - - ret = PIO_enddef(pio_file) - PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) - - call PIO_closefile(pio_file) - call PIO_deletefile(pio_tf_iosystem_, filename); - end do - if(allocated(iotypes)) then - deallocate(iotypes) - deallocate(iotype_descs) - end if - -PIO_TF_AUTO_TEST_SUB_END test_put_1dval - -PIO_TF_TEMPLATE -PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dval - Implicit none - type(file_desc_t) :: pio_file - character(len=PIO_TF_MAX_STR_LEN) :: filename - type(var_desc_t) :: pio_var, pio_cvar - integer :: pio_dim - integer, parameter :: DIM_LEN = 100 - PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval - PIO_TF_FC_DATA_TYPE :: init_val - CHARACTER(len=DIM_LEN) :: pcval, gcval - integer, dimension(:), allocatable :: iotypes - character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs - integer :: num_iotypes - integer :: i, ret - - init_val = pio_tf_world_sz_ - - pval = init_val - pcval = "DUMMY_STRING" - num_iotypes = 0 - call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) - filename = "test_pio_ncdf_get_put.testfile" - do i=1,num_iotypes - PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) - ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) - PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) - - ! Since file is just created no need to enter redef - ret = PIO_def_dim(pio_file, 'dummy_dim', DIM_LEN, pio_dim) - PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) - - ret = PIO_def_var(pio_file, 'dummy_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) - PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) - - ret = PIO_def_var(pio_file, 'dummy_cval', PIO_char, (/pio_dim/), pio_cvar) - PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename)) - - ret = PIO_put_att(pio_file, pio_var, 'dummy_att_val', pval); - PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(filename)) - - ret = PIO_put_att(pio_file, pio_cvar, 'dummy_att_cval', pcval); - PIO_TF_CHECK_ERR(ret, "Failed to put char attribute:" // trim(filename)) - - ret = PIO_enddef(pio_file) - PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) - - ret = PIO_get_att(pio_file, pio_var, 'dummy_att_val', gval); - PIO_TF_CHECK_ERR(ret, "Failed to get attribute:" // trim(filename)) - - PIO_TF_CHECK_VAL((gval, init_val), "Got wrong value") - - ! FIXME: Check the values are correct - ret = PIO_get_att(pio_file, pio_cvar, 'dummy_att_cval', gcval); - PIO_TF_CHECK_ERR(ret, "Failed to get attribute:" // trim(filename)) - - call PIO_closefile(pio_file) - call PIO_deletefile(pio_tf_iosystem_, filename); - end do - if(allocated(iotypes)) then - deallocate(iotypes) - deallocate(iotype_descs) - end if - -PIO_TF_AUTO_TEST_SUB_END test_put_get_1dval - diff --git a/externals/pio2/tests/general/pio_decomp_tests.F90.in b/externals/pio2/tests/general/pio_decomp_tests.F90.in deleted file mode 100644 index 0bc10675a1e..00000000000 --- a/externals/pio2/tests/general/pio_decomp_tests.F90.in +++ /dev/null @@ -1,251 +0,0 @@ -PIO_TF_AUTO_TEST_SUB_BEGIN init_decomp_1d_get_loc_sz - implicit none - integer, parameter :: VEC_LOCAL_SZ = 7 - integer, dimension(:), allocatable :: data_types - character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: data_type_descs - type(io_desc_t) :: iodesc - integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps - integer, dimension(1) :: dims - integer :: i, ntypes, lsz - - do i=1,VEC_LOCAL_SZ - compdof_rel_disps(i) = i - end do - dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ - compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps - - call PIO_TF_Get_data_types(data_types, data_type_descs, ntypes) - do i=1,ntypes - PIO_TF_LOG(0,*) "Testing type : ", data_type_descs(i) - call PIO_initdecomp(pio_tf_iosystem_, data_types(i), dims, compdof, iodesc) - lsz = PIO_get_local_array_size(iodesc) - PIO_TF_PASSERT(lsz == VEC_LOCAL_SZ, "Checking the local array size") - call PIO_freedecomp(pio_tf_iosystem_, iodesc) - end do - - if(allocated(data_types)) then - deallocate(data_types) - deallocate(data_type_descs) - end if -PIO_TF_AUTO_TEST_SUB_END init_decomp_1d_get_loc_sz - -PIO_TF_TEMPLATE -PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_darray - implicit none - integer, parameter :: VEC_LOCAL_SZ = 7 - type(var_desc_t) :: pio_var - type(file_desc_t) :: pio_file - character(len=PIO_TF_MAX_STR_LEN) :: filename - type(io_desc_t) :: iodesc - integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps - PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: buf - integer, dimension(1) :: dims - integer :: pio_dim - integer :: i, ierr, lsz - ! iotypes = valid io types - integer, dimension(:), allocatable :: iotypes - character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs - integer :: num_iotypes - - do i=1,VEC_LOCAL_SZ - compdof_rel_disps(i) = i - end do - dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ - compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps - buf = pio_tf_world_rank_; - - call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) - - num_iotypes = 0 - call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) - filename = "test_pio_decomp_simple_tests.testfile" - do i=1,num_iotypes - PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) - PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) - - ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) - PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) - - ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) - PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) - - ierr = PIO_enddef(pio_file) - PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) - - ! Write the variable out - call PIO_write_darray(pio_file, pio_var, iodesc, buf, ierr) - PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) - - ! FIXME: Verify the written output - - call PIO_closefile(pio_file) - call PIO_deletefile(pio_tf_iosystem_, filename); - end do - if(allocated(iotypes)) then - deallocate(iotypes) - deallocate(iotype_descs) - end if - - call PIO_freedecomp(pio_tf_iosystem_, iodesc) -PIO_TF_AUTO_TEST_SUB_END nc_write_1d_darray - -! Get a block cyclic decomposition -! If force_rearrange is FALSE, the decomposition is such that -! # All even procs have VEC_LOCAL_SZ elements -! # All odd procs have VEC_LOCAL_SZ + 1 elements -! e.g. For VEC_LOCAL_SZ = 2, -! e.g. 1) [1,2] [3,4,5] [6,7] -! e.g. 2) [1,2] [3,4,5] [6,7] [8,9,10] -! e.g. 3) [1,2] [3,4,5] [6,7] [8,9,10] [11,12] -! If force_rearrange is TRUE, the decomposition is such that, -! If possible, the even rank "exchanges" elements with the next -! higher ranked odd proc. -! This for example can be used to force rearrangement when reading -! or writing data. -! e.g. For VEC_LOCAL_SZ = 2, -! e.g. 1) [3,4,5] [1,2] [6,7] -! e.g. 2) [3,4,5] [1,2] [8,9,10] [6,7] -! e.g. 3) [3,4,5] [1,2] [8,9,10] [6,7] [11,12] -SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange) - integer, parameter :: VEC_LOCAL_SZ = 7 - integer, intent(in) :: rank - integer, intent(in) :: sz - integer, dimension(1), intent(out) :: dims - integer, dimension(1), intent(out) :: start - integer, dimension(1), intent(out) :: count - logical, intent(in) :: force_rearrange - - logical :: is_even_rank - integer :: num_odd_procs, num_even_procs - integer :: iodd, ieven - - is_even_rank = .false. - if (mod(rank, 2) == 0) then - is_even_rank = .true. - end if - num_odd_procs = sz / 2 - num_even_procs = sz - num_odd_procs - dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1) - ! Number of odd and even procs before this rank - iodd = rank / 2 - ieven = (rank + 1) / 2 - if(force_rearrange) then - ! Make sure that we force rearrangement - if (is_even_rank) then - if(rank + 1 < sz) then - ! Force rearrangement - count(1) = VEC_LOCAL_SZ + 1 - start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1 - else - count(1) = VEC_LOCAL_SZ - start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1 - end if - else - ! For all odd procs there is an even lower ranked, rank-1, proc - ! So force rearrangement - count(1) = VEC_LOCAL_SZ - start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1 - end if - else - if (is_even_rank) then - count(1) = VEC_LOCAL_SZ - else - count(1) = VEC_LOCAL_SZ + 1 - end if - start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1 - end if - -END SUBROUTINE - -! Test block cyclic interface -! Write with one decomp (to force rearrangement) and read with another (no -! rearrangement) -PIO_TF_TEMPLATE -PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_1d_bc - implicit none - type(var_desc_t) :: pio_var - type(file_desc_t) :: pio_file - character(len=PIO_TF_MAX_STR_LEN) :: filename - type(io_desc_t) :: wr_iodesc, rd_iodesc - integer, dimension(:), allocatable :: compdof - integer, dimension(1) :: start, count - PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val - integer, dimension(1) :: dims - integer :: pio_dim - integer :: i, ierr, lsz - ! iotypes = valid io types - integer, dimension(:), allocatable :: iotypes - character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs - integer :: num_iotypes - - ! Set the decomposition for writing data - forcing rearrangement - call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.) - allocate(wbuf(count(1))) - allocate(compdof(count(1))) - do i=1,count(1) - wbuf(i) = start(1) + i - 1 - compdof(i) = start(1) + i - 1 - end do - - call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc) - deallocate(compdof) - - ! Set the decomposition for reading data - different from the write decomp - call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.) - allocate(rbuf(count(1))) - allocate(compdof(count(1))) - allocate(exp_val(count(1))) - do i=1,count(1) - compdof(i) = start(1) + i -1 - ! Expected value, after reading, is the same as the compdof - exp_val(i) = compdof(i) - end do - - call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc) - deallocate(compdof) - - num_iotypes = 0 - call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) - filename = "test_pio_decomp_simple_tests.testfile" - do i=1,num_iotypes - PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) - PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) - - ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) - PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) - - ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) - PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) - - ierr = PIO_enddef(pio_file) - PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) - - ! Write the variable out - call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr) - PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) - - call PIO_syncfile(pio_file) - - call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr) - PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) - - PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val") - - call PIO_closefile(pio_file) - - call PIO_deletefile(pio_tf_iosystem_, filename); - end do - - if(allocated(iotypes)) then - deallocate(iotypes) - deallocate(iotype_descs) - end if - - call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc) - call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc) - deallocate(exp_val) - deallocate(rbuf) - deallocate(wbuf) -PIO_TF_AUTO_TEST_SUB_END nc_write_read_1d_bc diff --git a/externals/pio2/tests/unit/CMakeLists.txt b/externals/pio2/tests/unit/CMakeLists.txt deleted file mode 100644 index 66d8995f7af..00000000000 --- a/externals/pio2/tests/unit/CMakeLists.txt +++ /dev/null @@ -1,91 +0,0 @@ -include (LibMPI) - -# Compiler-specific compiler options -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99") -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") -elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") -endif() -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") - -#============================================================================== -# PREPARE FOR TESTING -#============================================================================== - -file (COPY "./input.nl" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -file (COPY "./not_netcdf.ieee" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - -#============================================================================== -# DEFINE THE TARGETS AND TESTS -#============================================================================== - -set (SRCS basic_tests.F90 - driver.F90 - global_vars.F90 - ncdf_tests.F90 - nc_set_log_level2.c) - -add_executable (pio_unit_test EXCLUDE_FROM_ALL ${SRCS}) -target_link_libraries (pio_unit_test piof) -if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") - target_compile_options (pio_unit_test - PRIVATE -ffree-line-length-none) -endif() - -add_executable (test_names EXCLUDE_FROM_ALL test_names.c) -target_link_libraries (test_names pioc) -add_executable (test_nc4 EXCLUDE_FROM_ALL test_nc4.c) -target_link_libraries (test_nc4 pioc) - -if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") - set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) -# target_compile_options (gptl -# PRIVATE -mismatch_all) -endif () - -add_dependencies (tests test_names) -add_dependencies (tests test_nc4) -add_dependencies (tests pio_unit_test) - -# Test Timeout (4 min = 240 sec) -set (DEFAULT_TEST_TIMEOUT 240) - -if (PIO_USE_MPISERIAL) - add_test(NAME test_names - COMMAND test_names) - add_test(NAME test_nc4 - COMMAND test_nc4) - add_test(NAME pio_unit_test - COMMAND pio_unit_test) - set_tests_properties(pio_unit_test - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(test_names - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_names - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(test_nc4 - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_nc4 - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) - add_mpi_test(pio_unit_test - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_unit_test - NUMPROCS 4 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - -if (PIO_HDF5_LOGGING) - target_compile_definitions (pio_unit_test - PUBLIC LOGGING) -endif () diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 3e94c4db0e6..7fba3c18d61 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -7,8 +7,8 @@ Script to build a case. from standard_script_setup import * import CIME.build as build -from CIME.case import Case -from CIME.utils import append_status, find_system_test +from CIME.case import Case +from CIME.utils import find_system_test from CIME.XML.files import Files from CIME.XML.component import Component from CIME.test_status import * @@ -94,20 +94,10 @@ def _main_func(description): ts.set_status(phase_to_fail, TEST_FAIL_STATUS, comments="failed to initialize") raise - append_status("case.testbuild starting ", - caseroot=caseroot,sfile="CaseStatus") success = test.build(sharedlib_only=sharedlib_only, model_only=model_only) - if success: - append_status("case.testbuild complete", - caseroot=caseroot,sfile="CaseStatus") else: - append_status("case.build starting", - caseroot=caseroot,sfile="CaseStatus") success = build.case_build(caseroot, case=case, sharedlib_only=sharedlib_only, - model_only=model_only) - if success: - append_status("case.build complete", - caseroot=caseroot,sfile="CaseStatus") + model_only=model_only) sys.exit(0 if success else 1) diff --git a/scripts/Tools/case.setup b/scripts/Tools/case.setup index 2668d419aa2..ea60b98b1da 100755 --- a/scripts/Tools/case.setup +++ b/scripts/Tools/case.setup @@ -41,14 +41,20 @@ OR parser.add_argument("-t", "--test-mode", action="store_true", help="Keeps the test script when the --clean argument is used") + parser.add_argument("-r", "--reset", action="store_true", help="Does a clean followed by setup") + parser.add_argument("--no-adjust-pio", action="store_true", + help="Do NOT adjust pio settings for new pelayout." + "By default if the pelayout is changed the pio layout will be adjusted." + "This option overrides that feature and leaves the pio layout as currently set.") + args = parser.parse_args(args[1:]) CIME.utils.handle_standard_logging_options(args) - return args.caseroot, args.clean, args.test_mode, args.reset + return args.caseroot, args.clean, args.test_mode, args.reset, not args.no_adjust_pio ############################################################################### def _main_func(description): @@ -57,9 +63,9 @@ def _main_func(description): test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - caseroot, clean, test_mode, reset = parse_command_line(sys.argv, description) + caseroot, clean, test_mode, reset, adjust_pio = parse_command_line(sys.argv, description) with Case(caseroot, read_only=False) as case: - case_setup(case, clean, test_mode, reset) + case_setup(case, clean=clean, test_mode=test_mode, reset=reset, adjust_pio=adjust_pio) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/Tools/check_input_data b/scripts/Tools/check_input_data index 96a2dac0c5c..7795c063cac 100755 --- a/scripts/Tools/check_input_data +++ b/scripts/Tools/check_input_data @@ -41,8 +41,10 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--svn-loc", default=SVN_LOCS[get_model()], help="The input data repository from which to download data.") - parser.add_argument("-i", "--input-data-root", default=MACHINE.get_value("DIN_LOC_ROOT"), - help="The root directory where input data goes") + parser.add_argument("-i", "--input-data-root", default=None, + help="The root directory where input data goes. " + "Default value will come from DIN_LOC_ROOT from case which itself " + "defaults to '%s'" % MACHINE.get_value("DIN_LOC_ROOT")) parser.add_argument("--data-list-dir", default="Buildconf", help="Where to find list of input files") diff --git a/scripts/Tools/cime_bisect b/scripts/Tools/cime_bisect index 7cd85a115e4..6e2b37e93bd 100755 --- a/scripts/Tools/cime_bisect +++ b/scripts/Tools/cime_bisect @@ -88,7 +88,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter ############################################################################### def cime_bisect(testargs, good, bad, testroot, compiler, project, baseline_name, check_namelists, check_throughput, check_memory, all_commits): ############################################################################### - create_test = os.path.join(CIME.utils.get_scripts_location_within_cime(), "create_test") + create_test = os.path.join(CIME.utils.get_scripts_root(), "create_test") expect(os.path.exists(create_test), "Please run the root of a CIME repo") wait_for_tests = os.path.join(CIME.utils.get_scripts_location_within_cime(), "Tools", "wait_for_tests") diff --git a/scripts/Tools/standard_script_setup.py b/scripts/Tools/standard_script_setup.py index 2db8d738404..f95cd1c3257 100644 --- a/scripts/Tools/standard_script_setup.py +++ b/scripts/Tools/standard_script_setup.py @@ -7,7 +7,7 @@ import sys, os import __main__ as main _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..") -_LIB_DIR = os.path.join(_CIMEROOT, "utils", "python") +_LIB_DIR = os.path.join(_CIMEROOT, "scripts", "lib") sys.path.append(_LIB_DIR) # Important: Allows external tools to link up with CIME diff --git a/scripts/Tools/update_acme_tests b/scripts/Tools/update_acme_tests index dd86e620840..f5aeb4a0207 100755 --- a/scripts/Tools/update_acme_tests +++ b/scripts/Tools/update_acme_tests @@ -26,11 +26,11 @@ OR \033[1mEXAMPLES:\033[0m \033[1;32m# Update all acme test suites for all platforms \033[0m - > %s ../cime_config/acme/allactive/testlist_allactive.xml + > %s ../config/acme/allactive/testlist_allactive.xml \033[1;32m# Update acme_developer tests for all platforms \033[0m - > %s ../cime_config/acme/allactive/testlist_allactive.xml acme_developer + > %s ../config/acme/allactive/testlist_allactive.xml acme_developer \033[1;32m# Add acme_developer tests for a new platform \033[0m - > %s ../cime_config/acme/allactive/testlist_allactive.xml acme_developer -p machine,compiler + > %s ../config/acme/allactive/testlist_allactive.xml acme_developer -p machine,compiler """ % ((os.path.basename(args[0]), ) * 6), description=description, diff --git a/scripts/Tools/xmlchange b/scripts/Tools/xmlchange index 3607c35c9ba..57581e506f5 100755 --- a/scripts/Tools/xmlchange +++ b/scripts/Tools/xmlchange @@ -12,7 +12,7 @@ from standard_script_setup import * -from CIME.utils import expect, convert_to_type, append_status +from CIME.utils import expect, convert_to_type, append_case_status from CIME.case import Case import re @@ -135,7 +135,7 @@ def xmlchange(caseroot, listofsettings=None, xmlid=None, xmlval=None, subgroup=N for arg in sys.argv: argstr += "%s "%arg msg = " %s " % (argstr) - append_status(msg, caseroot=caseroot, sfile="CaseStatus") + append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot) def _main_func(description): if ("--test" in sys.argv): diff --git a/scripts/Tools/xmlconvertors/config_pes_converter.py b/scripts/Tools/xmlconvertors/config_pes_converter.py index cb1db6da482..a0d0746c227 100755 --- a/scripts/Tools/xmlconvertors/config_pes_converter.py +++ b/scripts/Tools/xmlconvertors/config_pes_converter.py @@ -5,7 +5,7 @@ The location of these files are needed by the script: CIME2: cime/machines-acme/config_pes.xml - CIME5: cime_config/acme/allactive/config_pesall.xml + CIME5: config/acme/allactive/config_pesall.xml """ diff --git a/scripts/Tools/xmlconvertors/grid_xml_converter.py b/scripts/Tools/xmlconvertors/grid_xml_converter.py index 4120d34ae16..28fb95f49b5 100755 --- a/scripts/Tools/xmlconvertors/grid_xml_converter.py +++ b/scripts/Tools/xmlconvertors/grid_xml_converter.py @@ -3,14 +3,14 @@ grid_xml_converter.py -- convert (or verify) grid xml from CIME2 format to CIME5 The location of these files are needed by the script: CIME2: cime/scripts/Tools/config_grid.xml - CIME5: cime_config/acme/config_grids.xml + CIME5: config/acme/config_grids.xml """ # make sure cime2, cime roots are defined # use categories # GRID CONFIGURATIONS grid list domain grid maps # CIME2: cime/scripts/Tools/config_grid.xml -# CIME5: cime_config/acme/config_grids.xml +# CIME5: config/acme/config_grids.xml # from standard_script_setup import * @@ -293,7 +293,7 @@ def postprocess(self, fixlist, addlist, newxmlfile, currentxmlfile, self.writexml(fixlist,badxmlfile) if len(fixlist) > 0: logger.info("Some nodes should be removed from") - logger.info("cime_config/acme/config_grids.xml. These nodes") + logger.info("config/acme/config_grids.xml. These nodes") logger.info("have been written to %s" % badxmlfile) class GridTree(DataTree): diff --git a/scripts/Tools/xmlquery b/scripts/Tools/xmlquery index fac6b8c53ee..0b3177004dc 100755 --- a/scripts/Tools/xmlquery +++ b/scripts/Tools/xmlquery @@ -129,7 +129,6 @@ def get_value_as_string(case, var, attribute=None, resolved=False, subgroup=None value = convert_to_string(value, thistype, var) return value - def xmlquery(case, variables, subgroup=None, fileonly=False, resolved=True, raw=False, description=False, group=False, full=False, dtype=False, valid_values=False): @@ -151,8 +150,9 @@ def xmlquery(case, variables, subgroup=None, fileonly=False, results[group] = {} if not var in results[group]: results[group][var] = {} + expect(group, "No group found for var %s"% var) - value = get_value_as_string(case, var, resolved=resolved, subgroup=subgroup) + value = get_value_as_string(case, var, resolved=resolved, subgroup=group) if value is None: var, comp, iscompvar = case.check_if_comp_var(var) if iscompvar: diff --git a/scripts/create_clone b/scripts/create_clone index bcc13a729d4..968848a7a42 100755 --- a/scripts/create_clone +++ b/scripts/create_clone @@ -25,7 +25,7 @@ def parse_command_line(args): "is assumed to be under then current working directory ") parser.add_argument("--keepexe", "-keepexe", action="store_true", - help="Sets EXEROOT to point to original build") + help="Sets EXEROOT to point to original build, it is highly recommended that the original case be built before cloning if using the --keepexe flag") parser.add_argument("--mach-dir", "-mach_dir", help="Specify the locations of the Machines directory, other than the default" diff --git a/scripts/create_newcase b/scripts/create_newcase index f0b969d87b5..f1877cd76e5 100755 --- a/scripts/create_newcase +++ b/scripts/create_newcase @@ -86,7 +86,7 @@ OR parser.add_argument("--gridfile", help="Full pathname of config grid file to use" - "This should be a copy of cime_config/config_grids.xml" + "This should be a copy of config/config_grids.xml" "with the new user grid changes added to it" "This argument is required if --user-grid is True") @@ -114,6 +114,10 @@ OR parser.add_argument("-q", "--queue", default=None, help="Force batch system to use a certain queue") + parser.add_argument("--handle-preexisting-dirs", dest="answer", choices=("a", "r", "u"), default=None, + help="Pre-answer what to do pre-existing bld/exe dirs. Valid answers are (a)bort (r)eplace or (u)se existing." + "This can be useful if you need to run create_newcase non-iteractively.") + args = parser.parse_args() CIME.utils.handle_standard_logging_options(args) @@ -143,12 +147,14 @@ OR if model == "cesm": run_unsupported = args.run_unsupported + expect(CIME.utils.check_name(args.case, fullpath=True), + "Illegal case name argument provided") return args.case, args.compset, args.res, args.machine, args.compiler,\ args.mpilib, args.project, args.pecount, \ args.user_mods_dir, args.user_compset, args.pesfile, \ args.user_grid, args.gridfile, args.srcroot, args.test, args.ninst, \ - args.walltime, args.queue, args.output_root, run_unsupported + args.walltime, args.queue, args.output_root, run_unsupported, args.answer ############################################################################### def _main_func(description): @@ -159,7 +165,7 @@ def _main_func(description): mpilib, project, pecount, \ user_mods_dir, user_compset, pesfile, \ user_grid, gridfile, srcroot, test, ninst, walltime, queue, \ - output_root, run_unsupported = parse_command_line(sys.argv, cimeroot, description) + output_root, run_unsupported, answer = parse_command_line(sys.argv, cimeroot, description) caseroot = os.path.abspath(caseroot) @@ -180,7 +186,7 @@ def _main_func(description): user_compset=user_compset, pesfile=pesfile, user_grid=user_grid, gridfile=gridfile, ninst=ninst, test=test, walltime=walltime, queue=queue, output_root=output_root, - run_unsupported=run_unsupported) + run_unsupported=run_unsupported, answer=answer) case.create_caseroot() diff --git a/scripts/create_test b/scripts/create_test index 1066e58ae7e..f1dcede9b6e 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -20,6 +20,7 @@ import argparse, math, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -222,6 +223,9 @@ OR parser.add_argument("--force-threads", type=int, default=None, help="For all tests to run with this number of threads") + parser.add_argument("-i", "--input-dir", + help="Use a non-default location for input files") + args = parser.parse_args(args[1:]) CIME.utils.handle_standard_logging_options(args) @@ -275,6 +279,9 @@ OR if args.test_id is None: args.test_id = "%s_%s"%(CIME.utils.get_timestamp(), CIME.utils.id_generator()) + else: + expect(CIME.utils.check_name(args.test_id, additional_chars="."), + "invalid test-id argument provided") if args.testfile is not None: with open(args.testfile, "r") as fd: @@ -343,11 +350,14 @@ OR elif args.generate: baseline_gen_name = baseline_name + if args.input_dir is not None: + args.input_dir = os.path.abspath(args.input_dir) + return test_names, test_extra_data, args.compiler, mach_obj.get_machine_name(), args.no_run, args.no_build, args.no_setup, args.no_batch,\ args.test_root, args.baseline_root, args.clean, baseline_cmp_name, baseline_gen_name, \ args.namelists_only, args.project, args.test_id, args.parallel_jobs, args.walltime, \ args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, \ - args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib + args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib, args.input_dir ############################################################################### def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost_map, wall_time, test_root): @@ -453,7 +463,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait, - force_procs, force_threads, mpilib): + force_procs, force_threads, mpilib, input_dir): ############################################################################### impl = TestScheduler(test_names, test_data=test_data, no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch, @@ -465,7 +475,8 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, project=project, parallel_jobs=parallel_jobs, walltime=walltime, proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing, queue=queue, allow_baseline_overwrite=allow_baseline_overwrite, - output_root=output_root, force_procs=force_procs, force_threads=force_threads, mpilib=mpilib) + output_root=output_root, force_procs=force_procs, force_threads=force_threads, + mpilib=mpilib, input_dir=input_dir) success = impl.run_tests(wait=wait) @@ -506,13 +517,13 @@ def _main_func(description): test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, \ test_root, baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, \ project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, \ - save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib \ + save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir \ = parse_command_line(sys.argv, description) sys.exit(create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing, - queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib)) + queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir)) ############################################################################### diff --git a/utils/data_assimilation/da_no_data_mod.sh b/scripts/data_assimilation/da_no_data_mod.sh similarity index 100% rename from utils/data_assimilation/da_no_data_mod.sh rename to scripts/data_assimilation/da_no_data_mod.sh diff --git a/utils/python/CIME/BuildTools/__init__.py b/scripts/lib/CIME/BuildTools/__init__.py similarity index 100% rename from utils/python/CIME/BuildTools/__init__.py rename to scripts/lib/CIME/BuildTools/__init__.py diff --git a/utils/python/CIME/BuildTools/cmakemacroswriter.py b/scripts/lib/CIME/BuildTools/cmakemacroswriter.py similarity index 100% rename from utils/python/CIME/BuildTools/cmakemacroswriter.py rename to scripts/lib/CIME/BuildTools/cmakemacroswriter.py diff --git a/utils/python/CIME/BuildTools/configure.py b/scripts/lib/CIME/BuildTools/configure.py similarity index 100% rename from utils/python/CIME/BuildTools/configure.py rename to scripts/lib/CIME/BuildTools/configure.py diff --git a/utils/python/CIME/BuildTools/macroconditiontree.py b/scripts/lib/CIME/BuildTools/macroconditiontree.py similarity index 100% rename from utils/python/CIME/BuildTools/macroconditiontree.py rename to scripts/lib/CIME/BuildTools/macroconditiontree.py diff --git a/utils/python/CIME/BuildTools/macrowriterbase.py b/scripts/lib/CIME/BuildTools/macrowriterbase.py similarity index 100% rename from utils/python/CIME/BuildTools/macrowriterbase.py rename to scripts/lib/CIME/BuildTools/macrowriterbase.py diff --git a/utils/python/CIME/BuildTools/makemacroswriter.py b/scripts/lib/CIME/BuildTools/makemacroswriter.py similarity index 100% rename from utils/python/CIME/BuildTools/makemacroswriter.py rename to scripts/lib/CIME/BuildTools/makemacroswriter.py diff --git a/utils/python/CIME/BuildTools/possiblevalues.py b/scripts/lib/CIME/BuildTools/possiblevalues.py similarity index 100% rename from utils/python/CIME/BuildTools/possiblevalues.py rename to scripts/lib/CIME/BuildTools/possiblevalues.py diff --git a/utils/python/CIME/BuildTools/valuesetting.py b/scripts/lib/CIME/BuildTools/valuesetting.py similarity index 100% rename from utils/python/CIME/BuildTools/valuesetting.py rename to scripts/lib/CIME/BuildTools/valuesetting.py diff --git a/utils/python/CIME/SystemTests/README b/scripts/lib/CIME/SystemTests/README similarity index 100% rename from utils/python/CIME/SystemTests/README rename to scripts/lib/CIME/SystemTests/README diff --git a/utils/python/CIME/SystemTests/__init__.py b/scripts/lib/CIME/SystemTests/__init__.py similarity index 100% rename from utils/python/CIME/SystemTests/__init__.py rename to scripts/lib/CIME/SystemTests/__init__.py diff --git a/utils/python/CIME/SystemTests/dae.py b/scripts/lib/CIME/SystemTests/dae.py similarity index 94% rename from utils/python/CIME/SystemTests/dae.py rename to scripts/lib/CIME/SystemTests/dae.py index 8a21f2ad853..9621d80e4e5 100644 --- a/utils/python/CIME/SystemTests/dae.py +++ b/scripts/lib/CIME/SystemTests/dae.py @@ -20,7 +20,7 @@ class DAE(SystemTestsCompareTwo): Implementation of the CIME data assimilation test: Tests simple DA script which does not change the CAM input. Compares answers to non-DA run. Refers to a faux data assimilation script in the - cime/utils/data_assimilation directory + cime/scripts/data_assimilation directory """ ########################################################################### @@ -41,7 +41,7 @@ def _case_one_setup(self): def _case_two_setup(self): ########################################################################### # Set up data assimilation in config_tests.xml once that's ready - # We need to find the utils/data_assimilation directory + # We need to find the scripts/data_assimilation directory # LIB_DIR should be our parent dir da_dir = os.path.join(os.path.dirname(sms.LIB_DIR), "data_assimilation") expect(os.path.isdir(da_dir), "ERROR: da_dir, '%s', does not exist"%da_dir) @@ -65,18 +65,21 @@ def run_phase(self): # Clean up any da.log files in case this is a re-run. self._activate_case2() case_root = self._get_caseroot2() - da_files = glob.glob(os.path.join(case_root, 'da.log.*')) + rundir2 = self._case.get_value("RUNDIR") + da_files = glob.glob(os.path.join(rundir2, 'da.log.*')) for file_ in da_files: os.remove(file_) # End for + # CONTINUE_RUN ends up TRUE, set it back in case this is a re-run. self._case.set_value("CONTINUE_RUN", False) # Start normal run here self._activate_case1() SystemTestsCompareTwo.run_phase(self) + # Do some checks on the data assimilation 'output' from case2 self._activate_case2() - da_files = glob.glob(os.path.join(case_root, 'da.log.*')) + da_files = glob.glob(os.path.join(rundir2, 'da.log.*')) if da_files is None: logger = logging.getLogger(__name__) logger.warning("No DA files in %s", os.path.join(case_root, 'da.log.*')) diff --git a/utils/python/CIME/SystemTests/eri.py b/scripts/lib/CIME/SystemTests/eri.py similarity index 100% rename from utils/python/CIME/SystemTests/eri.py rename to scripts/lib/CIME/SystemTests/eri.py diff --git a/utils/python/CIME/SystemTests/erio.py b/scripts/lib/CIME/SystemTests/erio.py similarity index 100% rename from utils/python/CIME/SystemTests/erio.py rename to scripts/lib/CIME/SystemTests/erio.py diff --git a/utils/python/CIME/SystemTests/erp.py b/scripts/lib/CIME/SystemTests/erp.py similarity index 98% rename from utils/python/CIME/SystemTests/erp.py rename to scripts/lib/CIME/SystemTests/erp.py index 2fbd4c87a61..9bade606e29 100644 --- a/utils/python/CIME/SystemTests/erp.py +++ b/scripts/lib/CIME/SystemTests/erp.py @@ -67,7 +67,6 @@ def build_phase(self, sharedlib_only=False, model_only=False): nthreads = self._case.get_value("NTHRDS_%s"%comp) rootpe = self._case.get_value("ROOTPE_%s"%comp) if ( nthreads > 1 ): - self._case.set_value("BUILD_THREADED", True) self._case.set_value("NTHRDS_%s"%comp, nthreads/2) if ( ntasks > 1 ): self._case.set_value("NTASKS_%s"%comp, ntasks/2) diff --git a/utils/python/CIME/SystemTests/err.py b/scripts/lib/CIME/SystemTests/err.py similarity index 100% rename from utils/python/CIME/SystemTests/err.py rename to scripts/lib/CIME/SystemTests/err.py diff --git a/utils/python/CIME/SystemTests/ers.py b/scripts/lib/CIME/SystemTests/ers.py similarity index 100% rename from utils/python/CIME/SystemTests/ers.py rename to scripts/lib/CIME/SystemTests/ers.py diff --git a/utils/python/CIME/SystemTests/ert.py b/scripts/lib/CIME/SystemTests/ert.py similarity index 100% rename from utils/python/CIME/SystemTests/ert.py rename to scripts/lib/CIME/SystemTests/ert.py diff --git a/utils/python/CIME/SystemTests/homme.py b/scripts/lib/CIME/SystemTests/homme.py similarity index 73% rename from utils/python/CIME/SystemTests/homme.py rename to scripts/lib/CIME/SystemTests/homme.py index 9334300b077..95269653344 100644 --- a/utils/python/CIME/SystemTests/homme.py +++ b/scripts/lib/CIME/SystemTests/homme.py @@ -4,9 +4,8 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.build import post_build -from CIME.utils import append_status +from CIME.utils import append_testlog from CIME.test_status import * - import shutil logger = logging.getLogger(__name__) @@ -34,10 +33,10 @@ def build_phase(self, sharedlib_only=False, model_only=False): gmake = self._case.get_value("GMAKE") basename = basegen if generate else basecmp - cmake_cmd = "cmake -C %s/components/homme/cmake/machineFiles/%s.cmake -DUSE_NUM_PROCS=%s %s/components/homme -DHOMME_BASELINE_DIR=%s/%s >& homme.bldlog" % (srcroot, mach, procs, srcroot, baseline, basename) + cmake_cmd = "cmake -C %s/components/homme/cmake/machineFiles/%s.cmake -DUSE_NUM_PROCS=%s %s/components/homme -DHOMME_BASELINE_DIR=%s/%s" % (srcroot, mach, procs, srcroot, baseline, basename) - run_cmd_no_fail(cmake_cmd, from_dir=exeroot) - run_cmd_no_fail("%s -j8 >> homme.bldlog 2>&1" % gmake, from_dir=exeroot) + run_cmd_no_fail(cmake_cmd, arg_stdout="homme.bldlog", combine_output=True, from_dir=exeroot) + run_cmd_no_fail("%s -j8" % gmake, arg_stdout="homme.bldlog", combine_output=True, from_dir=exeroot) post_build(self._case, [os.path.join(exeroot, "homme.bldlog")]) @@ -57,19 +56,25 @@ def run_phase(self): if generate: full_baseline_dir = os.path.join(baseline, basegen, "tests", "baseline") - run_cmd_no_fail("%s -j 4 baseline >& %s" % (gmake, log), from_dir=exeroot) - if os.path.isdir(full_baseline_dir): - shutil.rmtree(full_baseline_dir) - shutil.copytree(os.path.join(exeroot, "tests", "baseline"), full_baseline_dir) + stat = run_cmd("%s -j 4 baseline" % gmake, arg_stdout=log, combine_output=True, from_dir=exeroot)[0] + if stat == 0: + if os.path.isdir(full_baseline_dir): + shutil.rmtree(full_baseline_dir) + + shutil.copytree(os.path.join(exeroot, "tests", "baseline"), full_baseline_dir) + elif compare: - run_cmd_no_fail("%s -j 4 check >& %s" % (gmake, log), from_dir=exeroot) + stat = run_cmd("%s -j 4 check" % gmake, arg_stdout=log, combine_output=True, from_dir=exeroot)[0] + else: - run_cmd_no_fail("%s -j 4 baseline >& %s" % (gmake, log), from_dir=exeroot) + stat = run_cmd("%s -j 4 baseline" % gmake, arg_stdout=log, combine_output=True, from_dir=exeroot)[0] # Add homme.log output to TestStatus.log so that it can # appear on the dashboard. Otherwise, the TestStatus.log # is pretty useless for this test. - append_status(open(log, "r").read(), sfile="TestStatus.log") + append_testlog(open(log, "r").read()) + + expect(stat == 0, "RUN FAIL for HOMME") # Homme is a bit of an oddball test since it's not really running the ACME model # We need to override some methods to make the core infrastructure work. diff --git a/utils/python/CIME/SystemTests/icp.py b/scripts/lib/CIME/SystemTests/icp.py similarity index 100% rename from utils/python/CIME/SystemTests/icp.py rename to scripts/lib/CIME/SystemTests/icp.py diff --git a/utils/python/CIME/SystemTests/lii.py b/scripts/lib/CIME/SystemTests/lii.py similarity index 100% rename from utils/python/CIME/SystemTests/lii.py rename to scripts/lib/CIME/SystemTests/lii.py diff --git a/utils/python/CIME/SystemTests/nck.py b/scripts/lib/CIME/SystemTests/nck.py similarity index 100% rename from utils/python/CIME/SystemTests/nck.py rename to scripts/lib/CIME/SystemTests/nck.py diff --git a/utils/python/CIME/SystemTests/ncr.py b/scripts/lib/CIME/SystemTests/ncr.py similarity index 100% rename from utils/python/CIME/SystemTests/ncr.py rename to scripts/lib/CIME/SystemTests/ncr.py diff --git a/scripts/lib/CIME/SystemTests/nodefail.py b/scripts/lib/CIME/SystemTests/nodefail.py new file mode 100644 index 00000000000..2f0436a870e --- /dev/null +++ b/scripts/lib/CIME/SystemTests/nodefail.py @@ -0,0 +1,68 @@ +""" +CIME restart upon failed node test. +""" +from CIME.XML.standard_module_setup import * +from CIME.SystemTests.ers import ERS +from CIME.utils import get_model + +logger = logging.getLogger(__name__) + +class NODEFAIL(ERS): + + def __init__(self, case): + """ + initialize an object interface to the ERS system test + """ + ERS.__init__(self, case) + + self._fail_sentinel = os.path.join(case.get_value("RUNDIR"), "FAIL_SENTINEL") + self._fail_str = case.get_value("NODE_FAIL_REGEX") + + def _restart_fake_phase(self): + # Swap out model.exe for one that emits node failures + rundir = self._case.get_value("RUNDIR") + exeroot = self._case.get_value("EXEROOT") + fake_exe = \ +"""#!/bin/bash + +fail_sentinel=%s +cpl_log=%s/cpl.log.$LID +model_log=%s/%s.log.$LID +touch $cpl_log +touch $fail_sentinel +declare -i num_fails=$(cat $fail_sentinel | wc -l) +declare -i times_to_fail=${NODEFAIL_NUM_FAILS:-3} + +if ((num_fails < times_to_fail)); then + echo FAKE FAIL >> $cpl_log + echo FAIL >> $fail_sentinel + echo '%s' >> $model_log + sleep 1 + exit -1 +else + echo Insta pass + echo SUCCESSFUL TERMINATION > $cpl_log +fi +""" % (self._fail_sentinel, rundir, rundir, get_model(), self._fail_str) + + fake_exe_file = os.path.join(exeroot, "fake.sh") + with open(fake_exe_file, "w") as fd: + fd.write(fake_exe) + + os.chmod(fake_exe_file, 0755) + + prev_run_exe = self._case.get_value("run_exe") + env_mach_specific = self._case.get_env("mach_specific") + env_mach_specific.set_value("run_exe", fake_exe_file) + self._case.flush(flushall=True) + + self.run_indv(suffix=None) + + env_mach_specific = self._case.get_env("mach_specific") + env_mach_specific.set_value("run_exe", prev_run_exe) + self._case.flush(flushall=True) + + def run_phase(self): + self._ers_first_phase() + self._restart_fake_phase() + self._ers_second_phase() diff --git a/utils/python/CIME/SystemTests/pea.py b/scripts/lib/CIME/SystemTests/pea.py similarity index 100% rename from utils/python/CIME/SystemTests/pea.py rename to scripts/lib/CIME/SystemTests/pea.py diff --git a/utils/python/CIME/SystemTests/pem.py b/scripts/lib/CIME/SystemTests/pem.py similarity index 87% rename from utils/python/CIME/SystemTests/pem.py rename to scripts/lib/CIME/SystemTests/pem.py index 5257f3e8ad8..bd3c090a7de 100644 --- a/utils/python/CIME/SystemTests/pem.py +++ b/scripts/lib/CIME/SystemTests/pem.py @@ -26,8 +26,8 @@ def _case_one_setup(self): pass def _case_two_setup(self): - for comp in ['ATM','CPL','OCN','WAV','GLC','ICE','ROF','LND']: + for comp in self._case.get_values("COMP_CLASSES"): ntasks = self._case.get_value("NTASKS_%s"%comp) if ( ntasks > 1 ): self._case.set_value("NTASKS_%s"%comp, int(ntasks/2)) - case_setup(self._case, clean=False, test_mode=False, reset=True) + case_setup(self._case, reset=True) diff --git a/utils/python/CIME/SystemTests/pet.py b/scripts/lib/CIME/SystemTests/pet.py similarity index 93% rename from utils/python/CIME/SystemTests/pet.py rename to scripts/lib/CIME/SystemTests/pet.py index 99166a96ebf..d6a6718464b 100644 --- a/utils/python/CIME/SystemTests/pet.py +++ b/scripts/lib/CIME/SystemTests/pet.py @@ -14,8 +14,6 @@ class PET(SystemTestsCompareTwo): - _COMPONENT_LIST = ('ATM','CPL','OCN','WAV','GLC','ICE','ROF','LND') - def __init__(self, case): """ initialize a test object @@ -28,7 +26,7 @@ def __init__(self, case): def _case_one_setup(self): # first make sure that all components have threaded settings - for comp in self._COMPONENT_LIST: + for comp in self._case.get_values("COMP_CLASSES"): if self._case.get_value("NTHRDS_%s"%comp) <= 1: self._case.set_value("NTHRDS_%s"%comp, 2) @@ -37,7 +35,7 @@ def _case_one_setup(self): def _case_two_setup(self): #Do a run with all threads set to 1 - for comp in self._COMPONENT_LIST: + for comp in self._case.get_values("COMP_CLASSES"): self._case.set_value("NTHRDS_%s"%comp, 1) # The need for this is subtle. On batch systems, the entire PET test runs diff --git a/utils/python/CIME/SystemTests/pfs.py b/scripts/lib/CIME/SystemTests/pfs.py similarity index 100% rename from utils/python/CIME/SystemTests/pfs.py rename to scripts/lib/CIME/SystemTests/pfs.py diff --git a/utils/python/CIME/SystemTests/pre.py b/scripts/lib/CIME/SystemTests/pre.py similarity index 100% rename from utils/python/CIME/SystemTests/pre.py rename to scripts/lib/CIME/SystemTests/pre.py diff --git a/utils/python/CIME/SystemTests/rep.py b/scripts/lib/CIME/SystemTests/rep.py similarity index 100% rename from utils/python/CIME/SystemTests/rep.py rename to scripts/lib/CIME/SystemTests/rep.py diff --git a/utils/python/CIME/SystemTests/seq.py b/scripts/lib/CIME/SystemTests/seq.py similarity index 82% rename from utils/python/CIME/SystemTests/seq.py rename to scripts/lib/CIME/SystemTests/seq.py index 579b92ad471..c53eb12a864 100644 --- a/utils/python/CIME/SystemTests/seq.py +++ b/scripts/lib/CIME/SystemTests/seq.py @@ -3,7 +3,7 @@ """ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.case_setup import case_setup +from CIME.case_setup import case_setup, adjust_pio_layout from CIME.check_lockedfiles import * import shutil @@ -45,18 +45,19 @@ def build_phase(self, sharedlib_only=False, model_only=False): for comp in comp_classes: self._case.set_value("ROOTPE_%s"%comp, 0) else: - rootpe = 2 + totalpes = self._case.get_value("TOTALPES") + newntasks = max(1, totalpes/len(comp_classes)) + rootpe = newntasks + for comp in comp_classes: # here we set the cpl to have the first 2 tasks # and each component to have a different ROOTPE if comp == "CPL": - self._case.set_value("NTASKS_CPL", 2) + self._case.set_value("NTASKS_CPL", newntasks) else: - ntasks = self._case.get_value("NTASKS_%s"%comp) - if ntasks > 1: - self._case.set_value("NTASKS_%s"%comp, max(1,ntasks-rootpe)) - self._case.set_value("ROOTPE_%s"%comp, rootpe) - rootpe += 1 + self._case.set_value("NTASKS_%s"%comp, newntasks) + self._case.set_value("ROOTPE_%s"%comp, rootpe) + rootpe += newntasks self._case.flush() case_setup(self._case, test_mode=True, reset=True) self.clean_build() @@ -86,6 +87,11 @@ def run_phase(self): shutil.copy("%s/%s.exe.SEQ1"%(exeroot,cime_model), "%s/%s.exe"%(exeroot,cime_model)) restore("env_mach_pes.SEQ1.xml", newname="env_mach_pes.xml") + + # update the pelayout settings for this run + self._case.read_xml() + adjust_pio_layout(self._case, self._case.get_value("PES_PER_NODE")) + self.run_indv() restore("env_mach_pes.SEQ2.xml", newname="env_mach_pes.xml") @@ -95,5 +101,9 @@ def run_phase(self): "%s/%s.exe"%(exeroot,cime_model)) logger.info("doing a second %d %s test with rootpes set to zero" % (stop_n, stop_option)) + # update the pelayout settings for this run + self._case.read_xml() + adjust_pio_layout(self._case, self._case.get_value("PES_PER_NODE")) + self.run_indv(suffix="seq") self._component_compare_test("base", "seq") diff --git a/utils/python/CIME/SystemTests/sms.py b/scripts/lib/CIME/SystemTests/sms.py similarity index 100% rename from utils/python/CIME/SystemTests/sms.py rename to scripts/lib/CIME/SystemTests/sms.py diff --git a/utils/python/CIME/SystemTests/ssp.py b/scripts/lib/CIME/SystemTests/ssp.py similarity index 97% rename from utils/python/CIME/SystemTests/ssp.py rename to scripts/lib/CIME/SystemTests/ssp.py index cd6f2c18d41..1e240b3e39a 100644 --- a/utils/python/CIME/SystemTests/ssp.py +++ b/scripts/lib/CIME/SystemTests/ssp.py @@ -52,6 +52,7 @@ def run_phase(self): logger.info(" short term archiving is on ") clone.set_value("CLM_ACCELERATED_SPINUP", "on") + clone.set_value("RTM_MODE", "NULL") clone.set_value("STOP_N",stop_n1) clone.flush() @@ -83,6 +84,7 @@ def run_phase(self): self._case.set_value("RUN_TYPE", "hybrid") self._case.set_value("GET_REFCASE", False) self._case.set_value("RUN_REFCASE", "%s.ref1" % orig_casevar) + self._case.set_value("RTM_MODE", "NULL") self._case.set_value("RUN_REFDATE", refdate) self._case.set_value("STOP_N", stop_n2) diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py similarity index 92% rename from utils/python/CIME/SystemTests/system_tests_common.py rename to scripts/lib/CIME/SystemTests/system_tests_common.py index 826a9b73705..efade3a6783 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -3,7 +3,7 @@ """ from CIME.XML.standard_module_setup import * from CIME.XML.env_run import EnvRun -from CIME.utils import append_status +from CIME.utils import append_testlog from CIME.case_setup import case_setup from CIME.case_run import case_run from CIME.case_st_archive import case_st_archive @@ -65,7 +65,7 @@ def _resetup_case(self, phase): logging.warning("Resetting case due to detected re-run of phase %s" % phase) self._case.set_initial_test_values() - case_setup(self._case, reset=True, test_mode=True, no_status=True) + case_setup(self._case, reset=True, test_mode=True) def build(self, sharedlib_only=False, model_only=False): """ @@ -89,7 +89,7 @@ def build(self, sharedlib_only=False, model_only=False): success = False excmsg = "Exception during build:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc()) logger.warning(excmsg) - append_status(excmsg, sfile="TestStatus.log") + append_testlog(excmsg) time_taken = time.time() - start_time with self._test_status: @@ -130,8 +130,6 @@ def run(self): success = True start_time = time.time() try: - expect(self._test_status.get_status(MODEL_BUILD_PHASE) == TEST_PASS_STATUS, - "Model was not built!") self._resetup_case(RUN_PHASE) with self._test_status: self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS) @@ -148,9 +146,15 @@ def run(self): except: success = False - excmsg = "Exception during run:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc()) + msg = sys.exc_info()[1].message + if "RUN FAIL" in msg: + # Don't want to print stacktrace for a model failure since that + # is not a CIME/infrastructure problem. + excmsg = msg + else: + excmsg = "Exception during run:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc()) logger.warning(excmsg) - append_status(excmsg, sfile="TestStatus.log") + append_testlog(excmsg) # Writing the run status should be the very last thing due to wait_for_tests time_taken = time.time() - start_time @@ -225,7 +229,7 @@ def _coupler_log_indicates_run_complete(self): def _component_compare_copy(self, suffix): comments = copy(self._case, suffix) - append_status(comments, sfile="TestStatus.log") + append_testlog(comments) def _component_compare_test(self, suffix1, suffix2): """ @@ -233,7 +237,7 @@ def _component_compare_test(self, suffix1, suffix2): run case needs indirection based on success. """ success, comments = compare_test(self._case, suffix1, suffix2) - append_status(comments, sfile="TestStatus.log") + append_testlog(comments) status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS with self._test_status: self._test_status.set_status("%s_%s_%s" % (COMPARE_PHASE, suffix1, suffix2), status) @@ -300,7 +304,7 @@ def _check_for_memleak(self): self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) else: comment = "memleak detected, memory went from %f to %f in %d days" % (originalmem, finalmem, finaldate-originaldate) - append_status(comment, sfile="TestStatus.log") + append_testlog(comment) self._test_status.set_status(MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment) def compare_env_run(self, expected=None): @@ -313,10 +317,8 @@ def compare_env_run(self, expected=None): logging.warn(" Resetting %s for test" % key) f1obj.set_value(key, f2obj.get_value(key, resolved=False)) else: - print "Found difference in %s: case: %s original value %s" %\ + print "WARNING: Found difference in test %s: case: %s original value %s" %\ (key, diffs[key][0], diffs[key][1]) - print " Use option --force to run the test with this"\ - " value or --reset to reset to original" return False return True @@ -339,7 +341,7 @@ def _compare_baseline(self): with self._test_status: # compare baseline success, comments = compare_baseline(self._case) - append_status(comments, sfile="TestStatus.log") + append_testlog(comments) status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS ts_comments = comments if "\n" not in comments else None self._test_status.set_status(BASELINE_PHASE, status, comments=ts_comments) @@ -363,11 +365,11 @@ def _compare_baseline(self): else: comment = "Error: Memory usage increase > 10% from baseline" self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment) - append_status(comment, sfile="TestStatus.log") + append_testlog(comment) else: comment = "Error: Could not determine baseline memory usage" self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment) - append_status(comment, sfile="TestStatus.log") + append_testlog(comment) # compare throughput to baseline current = self._get_throughput(newestcpllogfile) @@ -380,7 +382,7 @@ def _compare_baseline(self): else: comment = "Error: Computation time increase > 25% from baseline" self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment) - append_status(comment, sfile="TestStatus.log") + append_testlog(comment) def _generate_baseline(self): """ @@ -389,7 +391,7 @@ def _generate_baseline(self): with self._test_status: # generate baseline success, comments = generate_baseline(self._case) - append_status(comments, sfile="TestStatus.log") + append_testlog(comments) status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS self._test_status.set_status("%s" % GENERATE_PHASE, status) basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) @@ -432,9 +434,6 @@ def run_indv(self, suffix='base', st_archive=False): os.environ["MPI_SHEPHERD"] = "true" super(FakeTest, self).run_indv(suffix, st_archive) - - - class TESTRUNPASS(FakeTest): def build_phase(self, sharedlib_only=False, model_only=False): @@ -445,7 +444,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): """ echo Insta pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID -cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +cp %s/scripts/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc """ % (rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, @@ -468,9 +467,9 @@ def build_phase(self, sharedlib_only=False, model_only=False): echo Insta pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID if [ -z "$TESTRUNDIFF_ALTERNATE" ]; then - cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc + cp %s/scripts/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc else - cp %s/utils/python/tests/cpl.hi2.nc.test %s/%s.cpl.hi.0.nc + cp %s/scripts/tests/cpl.hi2.nc.test %s/%s.cpl.hi.0.nc fi """ % (rundir, cimeroot, rundir, case, cimeroot, rundir, case) self._set_script(script) @@ -487,8 +486,8 @@ def build_phase(self, sharedlib_only=False, model_only=False): """ echo Insta pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID -cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc.base -cp %s/utils/python/tests/cpl.hi2.nc.test %s/%s.cpl.hi.0.nc.rest +cp %s/scripts/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc.base +cp %s/scripts/tests/cpl.hi2.nc.test %s/%s.cpl.hi.0.nc.rest """ % (rundir, cimeroot, rundir, case, cimeroot, rundir, case) self._set_script(script) super(TESTTESTDIFF, self).build_phase(sharedlib_only=sharedlib_only, @@ -546,7 +545,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): sleep 300 echo Slow pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID -cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +cp %s/scripts/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc """ % (rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, @@ -557,12 +556,12 @@ def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") case = self._case.get_value("CASE") - testfile = os.path.join(cimeroot,"utils","python","tests","cpl.log.failmemleak.gz") + testfile = os.path.join(cimeroot,"scripts","tests","cpl.log.failmemleak.gz") script = \ """ echo Insta pass gunzip -c %s > %s/cpl.log.$LID -cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +cp %s/scripts/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc """ % (testfile, rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, @@ -573,12 +572,12 @@ def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") case = self._case.get_value("CASE") - testfile = os.path.join(cimeroot,"utils","python","tests","cpl.log.passmemleak.gz") + testfile = os.path.join(cimeroot,"scripts","tests","cpl.log.passmemleak.gz") script = \ """ echo Insta pass gunzip -c %s > %s/cpl.log.$LID -cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +cp %s/scripts/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc """ % (testfile, rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, diff --git a/utils/python/CIME/SystemTests/system_tests_compare_two.py b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py similarity index 97% rename from utils/python/CIME/SystemTests/system_tests_compare_two.py rename to scripts/lib/CIME/SystemTests/system_tests_compare_two.py index f51543dd79b..3eb1057373c 100644 --- a/utils/python/CIME/SystemTests/system_tests_compare_two.py +++ b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py @@ -144,6 +144,12 @@ def build_phase(self, sharedlib_only=False, model_only=False): else: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) + # pio_typename may be changed during the build if the default is not a + # valid value for this build, update case2 to reflect this change + for comp in self._case1.get_values("COMP_CLASSES"): + comp_pio_typename = "%s_PIO_TYPENAME"%comp + self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename)) + # The following is needed when _case_two_setup has a case_setup call # despite sharing the build (e.g., to change NTHRDS) self._case2.set_value("BUILD_COMPLETE",True) diff --git a/utils/python/CIME/SystemTests/test_utils/__init__.py b/scripts/lib/CIME/SystemTests/test_utils/__init__.py similarity index 100% rename from utils/python/CIME/SystemTests/test_utils/__init__.py rename to scripts/lib/CIME/SystemTests/test_utils/__init__.py diff --git a/utils/python/CIME/SystemTests/test_utils/user_nl_utils.py b/scripts/lib/CIME/SystemTests/test_utils/user_nl_utils.py similarity index 100% rename from utils/python/CIME/SystemTests/test_utils/user_nl_utils.py rename to scripts/lib/CIME/SystemTests/test_utils/user_nl_utils.py diff --git a/utils/python/CIME/XML/__init__.py b/scripts/lib/CIME/XML/__init__.py similarity index 100% rename from utils/python/CIME/XML/__init__.py rename to scripts/lib/CIME/XML/__init__.py diff --git a/utils/python/CIME/XML/archive.py b/scripts/lib/CIME/XML/archive.py similarity index 81% rename from utils/python/CIME/XML/archive.py rename to scripts/lib/CIME/XML/archive.py index ae5e291a3f3..24ed90356b5 100644 --- a/utils/python/CIME/XML/archive.py +++ b/scripts/lib/CIME/XML/archive.py @@ -6,7 +6,7 @@ from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files -from CIME.utils import expect +from CIME.utils import expect, get_model logger = logging.getLogger(__name__) @@ -26,8 +26,16 @@ def setup(self, env_archive, components, files=None): if files is None: files = Files() + components_node = ET.Element("components") + components_node.set("version", "2.0") + + + model = get_model() if 'cpl' not in components: components.append('cpl') + if 'dart' not in components and model == 'cesm': + components.append('dart') + for comp in components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp}) @@ -45,5 +53,6 @@ def setup(self, env_archive, components, files=None): logger.debug("No archive specs found for component %s"%comp) else: logger.debug("adding archive spec for %s"%comp) - env_archive.add_child(specs) + components_node.append(specs) + env_archive.add_child(components_node) diff --git a/utils/python/CIME/XML/batch.py b/scripts/lib/CIME/XML/batch.py similarity index 100% rename from utils/python/CIME/XML/batch.py rename to scripts/lib/CIME/XML/batch.py diff --git a/utils/python/CIME/XML/compilerblock.py b/scripts/lib/CIME/XML/compilerblock.py similarity index 100% rename from utils/python/CIME/XML/compilerblock.py rename to scripts/lib/CIME/XML/compilerblock.py diff --git a/utils/python/CIME/XML/compilers.py b/scripts/lib/CIME/XML/compilers.py similarity index 100% rename from utils/python/CIME/XML/compilers.py rename to scripts/lib/CIME/XML/compilers.py diff --git a/utils/python/CIME/XML/component.py b/scripts/lib/CIME/XML/component.py similarity index 100% rename from utils/python/CIME/XML/component.py rename to scripts/lib/CIME/XML/component.py diff --git a/utils/python/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py similarity index 98% rename from utils/python/CIME/XML/compsets.py rename to scripts/lib/CIME/XML/compsets.py index 7612899759e..875aea5d99a 100644 --- a/utils/python/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -38,7 +38,7 @@ def get_compset_match(self, name): logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support) - return (None, None, False) + return (None, None, [False]) def get_compset_var_settings(self, compset, grid): ''' @@ -85,4 +85,3 @@ def print_values(self): for v in compsets_text.iteritems(): label, definition = v logger.info(" %20s : %s" %(label, definition)) - diff --git a/utils/python/CIME/XML/entry_id.py b/scripts/lib/CIME/XML/entry_id.py similarity index 90% rename from utils/python/CIME/XML/entry_id.py rename to scripts/lib/CIME/XML/entry_id.py index 71ee53f833a..331ed21b0ba 100644 --- a/utils/python/CIME/XML/entry_id.py +++ b/scripts/lib/CIME/XML/entry_id.py @@ -130,7 +130,6 @@ def check_if_comp_var(self, vid, comp=None): # handled in classes return vid, None, False - def _get_default(self, node): return self._get_node_element_info(node, "default_value") @@ -152,7 +151,6 @@ def get_groups(self, node): return result - def get_valid_values(self, vid): node = self.get_optional_node("entry", {"id":vid}) if node is None: @@ -175,7 +173,6 @@ def set_valid_values(self, vid, new_valid_values): def get_nodes_by_id(self, vid): return self.get_nodes("entry", {"id":vid}) - def _set_valid_values(self, node, new_valid_values): old_vv = self._get_valid_values(node) if old_vv is None: @@ -201,18 +198,22 @@ def _set_value(self, node, value, vid=None, subgroup=None, ignore_type=False): subgroup is ignored in the general routine and applied in specific methods """ expect(subgroup is None, "Subgroup not supported") + str_value = self.get_valid_value_string(node, value, vid, ignore_type) + node.set("value", str_value) + return value + + def get_valid_value_string(self, node, value,vid=None, ignore_type=False): valid_values = self._get_valid_values(node) if ignore_type: expect(type(value) is str, "Value must be type string if ignore_type is true") str_value = value - else: - type_str = self._get_type_info(node) - str_value = convert_to_string(value, type_str, vid) + return str_value + type_str = self._get_type_info(node) + str_value = convert_to_string(value, type_str, vid) + if valid_values is not None and not str_value.startswith('$'): expect(str_value in valid_values, "Did not find %s in valid values for %s: %s" % (value, vid, valid_values)) - node.set("value", str_value) - - return value + return str_value def set_value(self, vid, value, subgroup=None, ignore_type=False): """ @@ -221,7 +222,8 @@ def set_value(self, vid, value, subgroup=None, ignore_type=False): subgroup is ignored in the general routine and applied in specific methods """ val = None - node = self.get_optional_node("entry", {"id":vid}) + root = self.root if subgroup is None else self.get_optional_node("group", {"id":subgroup}) + node = self.get_optional_node("entry", {"id":vid}, root=root) if node is not None: val = self._set_value(node, value, vid, subgroup, ignore_type) return val @@ -254,7 +256,8 @@ def get_value(self, vid, attribute=None, resolved=True, subgroup=None): or from the values field if the attribute argument is provided and matches """ - node = self.get_optional_node("entry", {"id":vid}) + root = self.root if subgroup is None else self.get_optional_node("group", {"id":subgroup}) + node = self.get_optional_node("entry", {"id":vid}, root=root) if node is None: return val = self._get_value(node, attribute=attribute, resolved=resolved, subgroup=subgroup) @@ -367,11 +370,22 @@ def compare_xml(self, other): f1nodes = self.get_nodes("entry") for node in f1nodes: vid = node.get("id") - f2val = other.get_value(vid, resolved=False) - if f2val is not None: + f2match = other.get_optional_node("entry", attributes={"id":vid}) + if f2match is not None: f1val = self.get_value(vid, resolved=False) - if f2val != f1val: - xmldiffs[vid] = [f1val, f2val] + if f1val is not None: + f2val = other.get_value(vid, resolved=False) + if f1val != f2val: + xmldiffs[vid] = [f1val, f2val] + else: + f1val = ET.tostring(node, method="text") + f2val = ET.tostring(f2match, method="text") + if f2val != f1val: + f1value_nodes = self.get_nodes("value", root=node) + for valnode in f1value_nodes: + f2valnode = other.get_node("value", root=f2match, attributes=valnode.attrib) + if f2valnode.text != valnode.text: + xmldiffs["%s:%s"%(vid,valnode.attrib)] = [valnode.text, f2valnode.text] return xmldiffs diff --git a/utils/python/CIME/XML/env_archive.py b/scripts/lib/CIME/XML/env_archive.py similarity index 93% rename from utils/python/CIME/XML/env_archive.py rename to scripts/lib/CIME/XML/env_archive.py index 536d0df222c..a9072b1744d 100644 --- a/utils/python/CIME/XML/env_archive.py +++ b/scripts/lib/CIME/XML/env_archive.py @@ -26,9 +26,8 @@ def __init__(self, case_root=None, infile="env_archive.xml"): else: fullpath = os.path.join(case_root, infile) - # Initialize self - # If env_archive.xml file does not exists in case directory read default from config - GenericXML.__init__(self, fullpath) + schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_archive.xsd") + GenericXML.__init__(self, fullpath, schema=schema) # The following creates the CASEROOT/env_archive.xml contents in self.root if not os.path.isfile(fullpath): diff --git a/utils/python/CIME/XML/env_base.py b/scripts/lib/CIME/XML/env_base.py similarity index 88% rename from utils/python/CIME/XML/env_base.py rename to scripts/lib/CIME/XML/env_base.py index 38e06316a13..774c034c0b4 100644 --- a/utils/python/CIME/XML/env_base.py +++ b/scripts/lib/CIME/XML/env_base.py @@ -1,16 +1,16 @@ """ -Base class for env files . This class inherits from EntryID.py +Base class for env files. This class inherits from EntryID.py """ import string from CIME.XML.standard_module_setup import * from CIME.XML.entry_id import EntryID from CIME.XML.headers import Headers -from CIME.utils import convert_to_type, convert_to_string +from CIME.utils import convert_to_type logger = logging.getLogger(__name__) class EnvBase(EntryID): - def __init__(self, case_root, infile): + def __init__(self, case_root, infile, schema=None): if case_root is None: case_root = os.getcwd() @@ -19,7 +19,7 @@ def __init__(self, case_root, infile): else: fullpath = os.path.join(case_root, infile) - EntryID.__init__(self, fullpath) + EntryID.__init__(self, fullpath, schema=schema) if not os.path.isfile(fullpath): headerobj = Headers() headernode = headerobj.get_header_node(os.path.basename(fullpath)) @@ -95,7 +95,8 @@ def set_value(self, vid, value, subgroup=None, ignore_type=False): """ vid, comp, iscompvar = self.check_if_comp_var(vid, None) val = None - node = self.get_optional_node("entry", {"id":vid}) + root = self.root if subgroup is None else self.get_optional_node("group", {"id":subgroup}) + node = self.get_optional_node("entry", {"id":vid}, root=root) if node is not None: if iscompvar and comp is None: # pylint: disable=no-member @@ -104,6 +105,7 @@ def set_value(self, vid, value, subgroup=None, ignore_type=False): else: val = self._set_value(node, value, vid, subgroup, ignore_type, component=comp) return val + # pylint: disable=arguments-differ def _set_value(self, node, value, vid=None, subgroup=None, ignore_type=False, component=None): if vid is None: @@ -112,10 +114,10 @@ def _set_value(self, node, value, vid=None, subgroup=None, ignore_type=False, co if iscompvar: attribute = {"component":component} - type_str = self._get_type_info(node) - val = self.set_element_text("value", convert_to_string(value, type_str, vid), attribute, root=node) - return val - val = EntryID._set_value(self, node, value, vid, subgroup, ignore_type) + str_value = self.get_valid_value_string(node, value, vid, ignore_type) + val = self.set_element_text("value", str_value, attribute, root=node) + else: + val = EntryID._set_value(self, node, value, vid, subgroup, ignore_type) return val def get_nodes_by_id(self, varid): diff --git a/utils/python/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py similarity index 86% rename from utils/python/CIME/XML/env_batch.py rename to scripts/lib/CIME/XML/env_batch.py index a6297e71be9..88fcdfc8f21 100644 --- a/utils/python/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -5,7 +5,7 @@ import re import math from CIME.XML.standard_module_setup import * -from CIME.utils import convert_to_type, format_time +from CIME.utils import format_time from CIME.XML.env_base import EnvBase from CIME.utils import transform_vars, get_cime_root from copy import deepcopy @@ -46,18 +46,18 @@ def set_value(self, item, value, subgroup=None, ignore_type=False): expect(False, "could not interpret format for wallclock time %s"%value) value = format_time(walltime_format, t_spec, value) - # allow the user to set all instances of item if subgroup is not provided + # allow the user to set item for all jobs if subgroup is not provided if subgroup is None: nodes = self.get_nodes("entry", {"id":item}) for node in nodes: self._set_value(node, value, vid=item, ignore_type=ignore_type) val = value else: - nodes = self.get_nodes("job", {"name":subgroup}) - for node in nodes: - vnode = self.get_optional_node("entry", {"id":item}, root=node) - if vnode is not None: - val = self._set_value(vnode, value, vid=item, ignore_type=ignore_type) + group = self.get_optional_node("group", {"id":subgroup}) + if group is not None: + node = self.get_optional_node("entry", {"id":item}, root=group) + if node is not None: + val = self._set_value(node, value, vid=item, ignore_type=ignore_type) return val @@ -77,20 +77,8 @@ def get_value(self, item, attribute=None, resolved=True, subgroup="case.run"): elif not nodes: value = EnvBase.get_value(self,item,attribute,resolved) else: - job_node = self.get_optional_node("job", {"name":subgroup}) - if job_node is not None: - node = self.get_optional_node("entry", {"id":item}, root=job_node) - if node is not None: - value = node.get("value") - - if resolved: - value = self.get_resolved_value(value) + value = EnvBase.get_value(self, item, attribute=attribute, resolved=resolved, subgroup=subgroup) - # Return value as right type if we were able to fully resolve - # otherwise, we have to leave as string. - if value is not None and "$" not in value: - type_str = self._get_type_info(node) - value = convert_to_type(value, type_str, item) return value def get_type_info(self, vid): @@ -106,36 +94,41 @@ def get_type_info(self, vid): return type_info def get_jobs(self): - jobs = [] - for node in self.get_nodes("job"): - name = node.get("name") - jobs.append(name) - return jobs - - def create_job_groups(self, bjobs): - # only the job_submission group is repeated - group = self.get_node("group", {"id":"job_submission"}) - # look to see if any jobs are already defined - cjobs = self.get_jobs() - childnodes = [] + groups = self.get_nodes("group") + results = [] + for group in groups: + if group.get("id") not in ["job_submission", "config_batch"]: + results.append(group.get("id")) + + return results - expect(len(cjobs)==0," Looks like job groups have already been created") + def create_job_groups(self, batch_jobs): + # Subtle: in order to support dynamic batch jobs, we need to remove the + # job_submission group and replace with job-based groups - for child in reversed(group): + orig_group = self.get_optional_node("group", {"id":"job_submission"}) + expect(orig_group, "Looks like job groups have already been created") + + childnodes = [] + for child in reversed(orig_group): childnodes.append(deepcopy(child)) - group.remove(child) + orig_group.remove(child) + + self.root.remove(orig_group) - for name,jdict in bjobs: - newjob = ET.Element("job") - newjob.set("name",name) + for name, jdict in batch_jobs: + new_job_group = ET.Element("group") + new_job_group.set("id", name) for field in jdict.keys(): val = jdict[field] - node = ET.SubElement(newjob, "entry", {"id":field,"value":val}) + node = ET.SubElement(new_job_group, "entry", {"id":field,"value":val}) tnode = ET.SubElement(node, "type") tnode.text = "char" + for child in childnodes: - newjob.append(deepcopy(child)) - group.append(newjob) + new_job_group.append(deepcopy(child)) + + self.root.append(new_job_group) def cleanupnode(self, node): if node.get("id") == "batch_system": @@ -188,14 +181,14 @@ def make_batch_script(self, input_template, job, case, total_tasks, tasks_per_no fd.write(output_text) os.chmod(job, os.stat(job).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - def set_job_defaults(self, bjobs, pesize=None, walltime=None, force_queue=None): + def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=None): if self.batchtype is None: self.batchtype = self.get_batch_system_type() if self.batchtype == 'none': return - for job, jsect in bjobs: + for job, jsect in batch_jobs: task_count = jsect["task_count"] if task_count is None or task_count == "default": task_count = pesize @@ -214,8 +207,8 @@ def set_job_defaults(self, bjobs, pesize=None, walltime=None, force_queue=None): self.get_default_queue() walltime = self._default_walltime - self.set_value( "JOB_WALLCLOCK_TIME", walltime , subgroup=job) - logger.debug("Job %s queue %s walltime %s"%(job, queue, walltime)) + self.set_value("JOB_WALLCLOCK_TIME", walltime, subgroup=job) + logger.debug("Job %s queue %s walltime %s" % (job, queue, walltime)) def get_batch_directives(self, case, job, raw=False): """ @@ -262,7 +255,7 @@ def get_submit_args(self, case, job): else: if name.startswith("$"): name = name[1:] - val = case.get_value(name,subgroup=job) + val = case.get_value(name, subgroup=job) if val is None: val = case.get_resolved_value(name) @@ -292,17 +285,18 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): alljobs = self.get_jobs() startindex = 0 jobs = [] + firstjob = job if job is not None: expect(job in alljobs, "Do not know about batch job %s"%job) startindex = alljobs.index(job) for index, job in enumerate(alljobs): - logger.debug( "Index %d job %s startindex %d"%(index, job, startindex)) + logger.debug( "Index %d job %s startindex %d" % (index, job, startindex)) if index < startindex: continue try: prereq = self.get_value('prereq', subgroup=job, resolved=False) - if prereq is None: + if prereq is None or job == firstjob: prereq = True else: prereq = case.get_resolved_value(prereq) @@ -366,8 +360,7 @@ def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=No else: # This is what we want longterm function_name = job.replace(".", "_") - success = locals()[function_name](case) - expect(success, "%s failed" % function_name) + locals()[function_name](case) return @@ -391,7 +384,7 @@ def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=No submitcmd += string + " " logger.info("Submitting job script %s"%submitcmd) - output = run_cmd_no_fail(submitcmd) + output = run_cmd_no_fail(submitcmd, combine_output=True) jobid = self.get_job_id(output) logger.info("Submitted job id is %s"%jobid) return jobid @@ -458,9 +451,3 @@ def get_nodes(self, nodename, attributes=None, root=None, xpath=None): else: nodes = EnvBase.get_nodes(self, nodename, attributes, root, xpath) return nodes - - def get_groups(self, root): - groups = EnvBase.get_groups(self, root) - if len(groups) == 1 and groups[0] == "job_submission": - groups = self.get_jobs() - return groups diff --git a/utils/python/CIME/XML/env_build.py b/scripts/lib/CIME/XML/env_build.py similarity index 74% rename from utils/python/CIME/XML/env_build.py rename to scripts/lib/CIME/XML/env_build.py index 7adae61be6a..fe7f812fda3 100644 --- a/utils/python/CIME/XML/env_build.py +++ b/scripts/lib/CIME/XML/env_build.py @@ -13,4 +13,5 @@ def __init__(self, case_root=None, infile="env_build.xml",components=None): """ initialize an object interface to file env_build.xml in the case directory """ - EnvBase.__init__(self, case_root, infile) + schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_entry_id.xsd") + EnvBase.__init__(self, case_root, infile, schema=schema) diff --git a/utils/python/CIME/XML/env_case.py b/scripts/lib/CIME/XML/env_case.py similarity index 74% rename from utils/python/CIME/XML/env_case.py rename to scripts/lib/CIME/XML/env_case.py index 146f3ecddb8..572f695c95f 100644 --- a/utils/python/CIME/XML/env_case.py +++ b/scripts/lib/CIME/XML/env_case.py @@ -13,4 +13,5 @@ def __init__(self, case_root=None, infile="env_case.xml", components=None): """ initialize an object interface to file env_case.xml in the case directory """ - EnvBase.__init__(self, case_root, infile) + schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_entry_id.xsd") + EnvBase.__init__(self, case_root, infile, schema=schema) diff --git a/utils/python/CIME/XML/env_mach_pes.py b/scripts/lib/CIME/XML/env_mach_pes.py similarity index 91% rename from utils/python/CIME/XML/env_mach_pes.py rename to scripts/lib/CIME/XML/env_mach_pes.py index 5e91a13add9..f7c49809783 100644 --- a/utils/python/CIME/XML/env_mach_pes.py +++ b/scripts/lib/CIME/XML/env_mach_pes.py @@ -44,7 +44,7 @@ def get_cost_pes(self, totaltasks, max_thread_count, machine=None): figure out the value of COST_PES which is the pe value used to estimate model cost """ pespn = self.get_value("PES_PER_NODE") - num_nodes = self.get_total_nodes(totaltasks, max_thread_count) + num_nodes = self.get_total_nodes(totaltasks, max_thread_count)[0] # This is hardcoded because on yellowstone by default we # run with 15 pes per node # but pay for 16 pes per node. See github issue #518 @@ -68,6 +68,12 @@ def get_tasks_per_node(self, total_tasks, max_thread_count): return tasks_per_node def get_total_nodes(self, total_tasks, max_thread_count): + """ + Return (num_active_nodes, num_spare_nodes) + """ tasks_per_node = self.get_tasks_per_node(total_tasks, max_thread_count) num_nodes = int(math.ceil(float(total_tasks) / tasks_per_node)) - return num_nodes + return num_nodes, self.get_spare_nodes(num_nodes) + + def get_spare_nodes(self, num_nodes): + return int(math.ceil(float(num_nodes) * (self.get_value("PCT_SPARE_NODES") / 100.0))) diff --git a/utils/python/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py similarity index 94% rename from utils/python/CIME/XML/env_mach_specific.py rename to scripts/lib/CIME/XML/env_mach_specific.py index ba012966ccb..b3efbb54bba 100644 --- a/utils/python/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -108,7 +108,7 @@ def list_modules(self): source_cmd = "" if (module_system == "module"): - return run_cmd_no_fail("%smodule list 2>&1" % source_cmd) + return run_cmd_no_fail("%smodule list" % source_cmd, combine_output=True) elif (module_system == "soft"): # Does soft really not provide this capability? return "" @@ -126,7 +126,7 @@ def save_all_env_info(self, filename): """ with open(filename, "w") as f: f.write(self.list_modules()) - run_cmd_no_fail("echo -e '\n' >> %s && env >> %s" % (filename, filename)) + run_cmd_no_fail("echo -e '\n' && env", arg_stdout=filename) def make_env_mach_specific_file(self, compiler, debug, mpilib, shell): modules_to_load = self._get_modules_for_case(compiler, debug, mpilib) @@ -321,7 +321,7 @@ def get_module_system_cmd_path(self, lang): cmd_nodes = self.get_optional_node("cmd_path", attributes={"lang":lang}) return cmd_nodes.text if cmd_nodes is not None else None - def get_mpirun(self, case, attribs, check_members=None, job="case.run"): + def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only=False): """ Find best match, return (executable, {arg_name : text}) """ @@ -372,16 +372,17 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run"): the_match = best_match if best_match is not None else default_match # Now that we know the best match, compute the arguments - arg_node = self.get_optional_node("arguments", root=the_match) - if arg_node is not None: - arg_nodes = self.get_nodes("arg", root=arg_node) - for arg_node in arg_nodes: - arg_value = transform_vars(arg_node.text, - case=case, - subgroup=job, - check_members=check_members, - default=arg_node.get("default")) - args[arg_node.get("name")] = arg_value + if not exe_only: + arg_node = self.get_optional_node("arguments", root=the_match) + if arg_node is not None: + arg_nodes = self.get_nodes("arg", root=arg_node) + for arg_node in arg_nodes: + arg_value = transform_vars(arg_node.text, + case=case, + subgroup=job, + check_members=check_members, + default=arg_node.get("default")) + args[arg_node.get("name")] = arg_value exec_node = self.get_node("executable", root=the_match) expect(exec_node is not None,"No executable found") diff --git a/utils/python/CIME/XML/env_run.py b/scripts/lib/CIME/XML/env_run.py similarity index 79% rename from utils/python/CIME/XML/env_run.py rename to scripts/lib/CIME/XML/env_run.py index 64edafa6769..9064841bed2 100644 --- a/utils/python/CIME/XML/env_run.py +++ b/scripts/lib/CIME/XML/env_run.py @@ -16,7 +16,9 @@ def __init__(self, case_root=None, infile="env_run.xml", components=None): self._components = components self._component_value_list = ["PIO_TYPENAME", "PIO_STRIDE", "PIO_REARRANGER", "PIO_NUMTASKS", "PIO_ROOT"] - EnvBase.__init__(self, case_root, infile) + schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_entry_id.xsd") + + EnvBase.__init__(self, case_root, infile, schema=schema) diff --git a/utils/python/CIME/XML/env_test.py b/scripts/lib/CIME/XML/env_test.py similarity index 100% rename from utils/python/CIME/XML/env_test.py rename to scripts/lib/CIME/XML/env_test.py diff --git a/utils/python/CIME/XML/files.py b/scripts/lib/CIME/XML/files.py similarity index 84% rename from utils/python/CIME/XML/files.py rename to scripts/lib/CIME/XML/files.py index 0fae48d4220..61b2894b579 100644 --- a/utils/python/CIME/XML/files.py +++ b/scripts/lib/CIME/XML/files.py @@ -16,11 +16,11 @@ def __init__(self): >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) - '$CIMEROOT/cime_config/config_headers.xml' + '$CIMEROOT/config/config_headers.xml' """ cimeroot = get_cime_root() - infile = os.path.join(cimeroot, "cime_config", get_model(), "config_files.xml") - schema = os.path.join(cimeroot, "cime_config", "xml_schemas", "entry_id.xsd") + infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") + schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema) def get_schema(self, nodename): diff --git a/utils/python/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py similarity index 99% rename from utils/python/CIME/XML/generic_xml.py rename to scripts/lib/CIME/XML/generic_xml.py index bfb3d626a08..d40736d6e9b 100644 --- a/utils/python/CIME/XML/generic_xml.py +++ b/scripts/lib/CIME/XML/generic_xml.py @@ -5,7 +5,6 @@ from CIME.XML.standard_module_setup import * from distutils.spawn import find_executable from xml.dom import minidom -from CIME.utils import expect, get_cime_root import getpass @@ -17,7 +16,6 @@ def __init__(self, infile=None, schema=None): """ Initialize an object """ - logger.debug("Initializing %s" , infile) self.tree = None @@ -41,6 +39,7 @@ def __init__(self, infile=None, schema=None): root = ET.Element("xml") self.root = ET.SubElement(root, "file") self.root.set("id", os.path.basename(infile)) + self.root.set("version", "2.0") self.tree = ET.ElementTree(root) def read(self, infile, schema=None): diff --git a/utils/python/CIME/XML/grids.py b/scripts/lib/CIME/XML/grids.py similarity index 84% rename from utils/python/CIME/XML/grids.py rename to scripts/lib/CIME/XML/grids.py index 0f16557b26f..351b40d254e 100644 --- a/utils/python/CIME/XML/grids.py +++ b/scripts/lib/CIME/XML/grids.py @@ -115,7 +115,7 @@ def _read_config_grids_v1(self, name, compset): expect (False, "grid '%s' is not supported, use manage_case to determine supported grids " %name) - def _read_config_grids_v2(self, name,compset): + def _read_config_grids_v2(self, name, compset): """ read config_grids.xml with version 2.0 schema """ @@ -410,6 +410,84 @@ def print_values(self, long_output=None): helptext = self.get_element_text("help") logger.info("%s " %helptext) + if self._version == 1.0: + self._print_values_v1(long_output=long_output) + elif self._version >= 2.0: + self._print_values_v2(long_output=long_output) + + def _print_values_v2(self, long_output=None): + + logger.info("%5s-------------------------------------------------------------" %("")) + logger.info("%10s default component grids:\n" %("")) + logger.info(" component compset value " ) + logger.info("%5s-------------------------------------------------------------" %("")) + default_nodes = self.get_nodes(nodename="model_grid_defaults") + for default_node in default_nodes: + grid_nodes = self.get_nodes(nodename="grid", root=default_node) + for grid_node in grid_nodes: + name = grid_node.get("name") + compset = grid_node.get("compset") + value = grid_node.text + logger.info(" %6s %15s %10s" %(name, compset, value)) + logger.info("%5s-------------------------------------------------------------" %("")) + + domains = {} + if long_output is not None: + domain_nodes = self.get_nodes(nodename="domain") + for domain_node in domain_nodes: + name = domain_node.get("name") + if name == 'null': + continue + desc = self.get_node("desc", root=domain_node).text + #support = self.get_optional_node("support", root=domain_node).text + files = "" + file_nodes = self.get_nodes("file", root=domain_node) + for file_node in file_nodes: + filename = file_node.text + mask_attrib = file_node.get("mask") + grid_attrib = file_node.get("grid") + files += "\n " + filename + if mask_attrib or grid_attrib: + files += " (only for" + if mask_attrib: + files += " mask: " + mask_attrib + if grid_attrib: + files += " grid match: " + grid_attrib + if mask_attrib or grid_attrib: + files += ")" + domains[name] = "\n %s with domain file(s): %s " %(desc, files) + + model_grid_nodes = self.get_nodes(nodename="model_grid") + for model_grid_node in model_grid_nodes: + alias = model_grid_node.get("alias") + compset = model_grid_node.get("compset") + not_compset = model_grid_node.get("not_compset") + restriction = "" + if compset: + restriction += "only for compsets that are %s " %compset + if not_compset: + restriction += "only for compsets that are not %s " %not_compset + if restriction: + logger.info("\n alias: %s (%s)" % (alias,restriction)) + else: + logger.info("\n alias: %s" % (alias)) + grid_nodes = self.get_nodes("grid", root=model_grid_node) + grids = "" + gridnames = [] + for grid_node in grid_nodes: + gridnames.append(grid_node.text) + grids += grid_node.get("name") + ":" + grid_node.text + " " + logger.info(" non-default grids are: %s" %grids) + mask_nodes = self.get_nodes("mask", root=model_grid_node) + for mask_node in mask_nodes: + logger.info(" mask is: %s" %(mask_node.text)) + if long_output is not None: + gridnames = set(gridnames) + for gridname in gridnames: + if gridname != "null": + logger.info (" %s" %(domains[gridname])) + + def _print_values_v1(self, long_output=None): # write out grid elements grid_nodes = self.get_nodes(nodename="grid") for grid_node in grid_nodes: diff --git a/utils/python/CIME/XML/headers.py b/scripts/lib/CIME/XML/headers.py similarity index 94% rename from utils/python/CIME/XML/headers.py rename to scripts/lib/CIME/XML/headers.py index 86005522463..1307c95c83f 100644 --- a/utils/python/CIME/XML/headers.py +++ b/scripts/lib/CIME/XML/headers.py @@ -15,7 +15,7 @@ def __init__(self,infile=None): >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) - '$CIMEROOT/cime_config/config_headers.xml' + '$CIMEROOT/config/config_headers.xml' """ if infile is None: files = Files() diff --git a/utils/python/CIME/XML/lt_archive.py b/scripts/lib/CIME/XML/lt_archive.py similarity index 100% rename from utils/python/CIME/XML/lt_archive.py rename to scripts/lib/CIME/XML/lt_archive.py diff --git a/utils/python/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py similarity index 91% rename from utils/python/CIME/XML/machines.py rename to scripts/lib/CIME/XML/machines.py index 804041fb465..cbe458e2afe 100644 --- a/utils/python/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -95,8 +95,34 @@ def probe_machine_name(self): Find a matching regular expression for hostname in the NODENAME_REGEX field in the file. First match wins. """ - machine = None + + names_not_found = [] + nametomatch = socket.getfqdn() + machine = self._probe_machine_name_one_guess(nametomatch) + + if machine is None: + names_not_found.append(nametomatch) + + nametomatch = socket.gethostname() + machine = self._probe_machine_name_one_guess(nametomatch) + + if machine is None: + names_not_found.append(nametomatch) + + names_not_found_quoted = ["'" + name + "'" for name in names_not_found] + names_not_found_str = ' or '.join(names_not_found_quoted) + logger.warning("Could not find machine match for %s" % names_not_found_str) + + return machine + + def _probe_machine_name_one_guess(self, nametomatch): + """ + Find a matching regular expression for nametomatch in the NODENAME_REGEX + field in the file. First match wins. Returns None if no match is found. + """ + + machine = None nodes = self.get_nodes("machine") for node in nodes: @@ -113,10 +139,6 @@ def probe_machine_name(self): machine = machtocheck break - if machine is None: - # Check for a local definition - logger.warning("Could not probe machine for hostname '%s'" % nametomatch) - return machine def set_machine(self, machine): diff --git a/utils/python/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py similarity index 95% rename from utils/python/CIME/XML/namelist_definition.py rename to scripts/lib/CIME/XML/namelist_definition.py index b9f76f2aaa8..dcb59bd2c85 100644 --- a/utils/python/CIME/XML/namelist_definition.py +++ b/scripts/lib/CIME/XML/namelist_definition.py @@ -11,10 +11,11 @@ # pylint:disable=wildcard-import,unused-wildcard-import import re +import collections from CIME.namelist import fortran_namelist_base_value, \ is_valid_fortran_namelist_literal, character_literal_to_string, \ - expand_literal_list, Namelist + expand_literal_list, Namelist, get_fortran_name_only from CIME.XML.standard_module_setup import * from CIME.XML.entry_id import EntryID @@ -306,7 +307,7 @@ def is_valid_value(self, name, value): # Check size of input array. if len(expand_literal_list(value)) > size: - return False + expect(False, "Value index exceeds variable size for variable %s, allowed array length is %s value array size is %s"%(name, size, len(expand_literal_list(value)))) return True def _expect_variable_in_definition(self, name, variable_template): @@ -342,21 +343,22 @@ def validate(self, namelist,filename=None): for group_name in namelist.get_group_names(): for variable_name in namelist.get_variable_names(group_name): # Check that the variable is defined... - self._expect_variable_in_definition(variable_name, variable_template) + qualified_variable_name = get_fortran_name_only(variable_name) + self._expect_variable_in_definition(qualified_variable_name, variable_template) # Check if can actually change this variable via filename change if filename is not None: - self._user_modifiable_in_variable_definition(variable_name) + self._user_modifiable_in_variable_definition(qualified_variable_name) # and has the right group name... - var_group = self.get_group(variable_name) + var_group = self.get_group(qualified_variable_name) expect(var_group == group_name, (variable_template + " is in a group named %r, but should be in %r.") % (str(variable_name), str(group_name), str(var_group))) # and has a valid value. value = namelist.get_variable_value(group_name, variable_name) - expect(self.is_valid_value(variable_name, value), + expect(self.is_valid_value(qualified_variable_name, value), (variable_template + " has invalid value %r.") % (str(variable_name), [str(scalar) for scalar in value])) @@ -379,11 +381,12 @@ def dict_to_namelist(self, dict_, filename=None): groups = {} for variable_name in dict_: variable_lc = variable_name.lower() - self._expect_variable_in_definition(variable_lc, variable_template) - group_name = self.get_group(variable_lc) + qualified_varname = get_fortran_name_only(variable_lc) + self._expect_variable_in_definition(qualified_varname, variable_template) + group_name = self.get_group(qualified_varname) expect (group_name is not None, "No group found for var %s"%variable_lc) if group_name not in groups: - groups[group_name] = {} + groups[group_name] = collections.OrderedDict() groups[group_name][variable_lc] = dict_[variable_name] return Namelist(groups) diff --git a/utils/python/CIME/XML/pes.py b/scripts/lib/CIME/XML/pes.py similarity index 100% rename from utils/python/CIME/XML/pes.py rename to scripts/lib/CIME/XML/pes.py diff --git a/utils/python/CIME/XML/pio.py b/scripts/lib/CIME/XML/pio.py similarity index 100% rename from utils/python/CIME/XML/pio.py rename to scripts/lib/CIME/XML/pio.py diff --git a/utils/python/CIME/XML/standard_module_setup.py b/scripts/lib/CIME/XML/standard_module_setup.py similarity index 76% rename from utils/python/CIME/XML/standard_module_setup.py rename to scripts/lib/CIME/XML/standard_module_setup.py index 75e2adacaab..8ab608c5db4 100644 --- a/utils/python/CIME/XML/standard_module_setup.py +++ b/scripts/lib/CIME/XML/standard_module_setup.py @@ -8,4 +8,4 @@ import re LIB_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(LIB_DIR) -from CIME.utils import expect, run_cmd, run_cmd_no_fail +from CIME.utils import expect, run_cmd, run_cmd_no_fail, get_cime_root diff --git a/utils/python/CIME/XML/testlist.py b/scripts/lib/CIME/XML/testlist.py similarity index 100% rename from utils/python/CIME/XML/testlist.py rename to scripts/lib/CIME/XML/testlist.py diff --git a/utils/python/CIME/XML/tests.py b/scripts/lib/CIME/XML/tests.py similarity index 100% rename from utils/python/CIME/XML/tests.py rename to scripts/lib/CIME/XML/tests.py diff --git a/utils/python/CIME/XML/testspec.py b/scripts/lib/CIME/XML/testspec.py similarity index 100% rename from utils/python/CIME/XML/testspec.py rename to scripts/lib/CIME/XML/testspec.py diff --git a/utils/python/CIME/__init__.py b/scripts/lib/CIME/__init__.py similarity index 100% rename from utils/python/CIME/__init__.py rename to scripts/lib/CIME/__init__.py diff --git a/utils/python/CIME/aprun.py b/scripts/lib/CIME/aprun.py similarity index 96% rename from utils/python/CIME/aprun.py rename to scripts/lib/CIME/aprun.py index a01d6fb15cb..cf5b9e218d6 100755 --- a/utils/python/CIME/aprun.py +++ b/scripts/lib/CIME/aprun.py @@ -30,10 +30,10 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, >>> machine = "titan" >>> run_exe = "acme.exe" >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) - 'aprun -S 4 -n 680 -N 8 -d 2 acme.exe : -S 2 -n 128 -N 4 -d 4 acme.exe ' + ('aprun -S 4 -n 680 -N 8 -d 2 acme.exe : -S 2 -n 128 -N 4 -d 4 acme.exe ', 117) >>> compiler = "intel" >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) - 'aprun -S 4 -cc numa_node -n 680 -N 8 -d 2 acme.exe : -S 2 -cc numa_node -n 128 -N 4 -d 4 acme.exe ' + ('aprun -S 4 -cc numa_node -n 680 -N 8 -d 2 acme.exe : -S 2 -cc numa_node -n 128 -N 4 -d 4 acme.exe ', 117) """ max_tasks_per_node = 1 if max_tasks_per_node < 1 else max_tasks_per_node diff --git a/utils/python/CIME/bless_test_results.py b/scripts/lib/CIME/bless_test_results.py similarity index 100% rename from utils/python/CIME/bless_test_results.py rename to scripts/lib/CIME/bless_test_results.py diff --git a/utils/python/CIME/build.py b/scripts/lib/CIME/build.py similarity index 90% rename from utils/python/CIME/build.py rename to scripts/lib/CIME/build.py index 62bd4e056a4..74b34c2ef0f 100644 --- a/utils/python/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -2,17 +2,17 @@ functions for building CIME models """ from CIME.XML.standard_module_setup import * -from CIME.utils import get_model, append_status, analyze_build_log, stringify_bool +from CIME.utils import get_model, analyze_build_log, stringify_bool, run_and_log_case_status from CIME.provenance import save_build_provenance -from CIME.preview_namelists import create_namelists +from CIME.preview_namelists import create_namelists, create_dirs from CIME.check_lockedfiles import check_lockedfiles, lock_file, unlock_file import glob, shutil, time, threading, gzip, subprocess logger = logging.getLogger(__name__) ############################################################################### -def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, - lid, caseroot, cimeroot, compiler): +def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, + lid, caseroot, cimeroot, compiler): ############################################################################### logs = [] @@ -29,7 +29,7 @@ def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, continue # special case for clm - # clm 4_0 is not a shared library and must be built here + # clm 4_0 is not a shared (as in sharedlibs, shared by all tests) library and must be built here # clm 4_5 and newer is a shared library (but not in ACME) and should be built in build_libraries if get_model() != "acme": if comp == "clm": @@ -38,7 +38,7 @@ def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, else: continue else: - logger.info(" - Building clm Library ") + logger.info(" - Building %s Library " % model) smp = nthrds > 1 or build_threaded @@ -90,7 +90,7 @@ def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, cime_model = get_model() file_build = os.path.join(exeroot, "%s.bldlog.%s" % (cime_model, lid)) - config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config") + config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config") f = open(file_build, "w") bldroot = os.path.join(exeroot, "cpl", "obj") if not os.path.isdir(bldroot): @@ -112,206 +112,9 @@ def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, return logs ############################################################################### -def post_build(case, logs): -############################################################################### - - logdir = case.get_value("LOGDIR") - - #zip build logs to CASEROOT/logs - if logdir: - bldlogdir = os.path.join(logdir, "bld") - if not os.path.exists(bldlogdir): - os.makedirs(bldlogdir) - - for log in logs: - logger.debug("Copying build log %s to %s"%(log,bldlogdir)) - with open(log, 'rb') as f_in: - with gzip.open("%s.gz"%log, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - if "sharedlibroot" not in log: - shutil.copy("%s.gz"%log,os.path.join(bldlogdir,"%s.gz"%os.path.basename(log))) - - # Set XML to indicate build complete - case.set_value("BUILD_COMPLETE", True) - case.set_value("BUILD_STATUS", 0) - if "SMP_VALUE" in os.environ: - case.set_value("SMP_BUILD", os.environ["SMP_VALUE"]) - case.flush() - - lock_file("env_build.xml") - - # must ensure there's an lid - lid = os.environ["LID"] if "LID" in os.environ else run_cmd_no_fail("date +%y%m%d-%H%M%S") - save_build_provenance(case, lid=lid) - -############################################################################### -def case_build(caseroot, case, sharedlib_only=False, model_only=False): -############################################################################### - - t1 = time.time() - - expect(not (sharedlib_only and model_only), - "Contradiction: both sharedlib_only and model_only") - logger.info("Building case in directory %s"%caseroot) - logger.info("sharedlib_only is %s" % sharedlib_only) - logger.info("model_only is %s" % model_only) - - expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot) - os.chdir(caseroot) - - expect(os.path.exists("case.run"), - "ERROR: must invoke case.setup script before calling build script ") - - cimeroot = case.get_value("CIMEROOT") - - comp_classes = case.get_values("COMP_CLASSES") - - check_lockedfiles(caseroot) - - # Retrieve relevant case data - # This environment variable gets set for cesm Make and - # needs to be unset before building again. - if "MODEL" in os.environ.keys(): - del os.environ["MODEL"] - build_threaded = case.get_value("BUILD_THREADED") - casetools = case.get_value("CASETOOLS") - exeroot = case.get_value("EXEROOT") - incroot = case.get_value("INCROOT") - libroot = case.get_value("LIBROOT") - sharedlibroot = case.get_value("SHAREDLIBROOT") - - complist = [] - for comp_class in comp_classes: - if comp_class == "CPL": - ninst = 1 - config_dir = None - else: - ninst = case.get_value("NINST_%s"%comp_class) - config_dir = os.path.dirname(case.get_value("CONFIG_%s_FILE"%comp_class)) - comp = case.get_value("COMP_%s"%comp_class) - thrds = case.get_value("NTHRDS_%s"%comp_class) - expect(ninst is not None,"Failed to get ninst for comp_class %s"%comp_class) - complist.append((comp_class.lower(), comp, thrds, ninst, config_dir )) - os.environ["COMP_%s"%comp_class] = comp - - ocn_submodel = case.get_value("OCN_SUBMODEL") - profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE") - compiler = case.get_value("COMPILER") - comp_interface = case.get_value("COMP_INTERFACE") - mpilib = case.get_value("MPILIB") - use_esmf_lib = case.get_value("USE_ESMF_LIB") - debug = case.get_value("DEBUG") - ninst_build = case.get_value("NINST_BUILD") - smp_value = case.get_value("SMP_VALUE") - clm_use_petsc = case.get_value("CLM_USE_PETSC") - cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") - mpasli_use_albany = case.get_value("MPASLI_USE_ALBANY") - clm_config_opts = case.get_value("CLM_CONFIG_OPTS") - cam_config_opts = case.get_value("CAM_CONFIG_OPTS") - pio_config_opts = case.get_value("PIO_CONFIG_OPTS") - ninst_value = case.get_value("NINST_VALUE") - mach = case.get_value("MACH") - os_ = case.get_value("OS") - # Load some params into env - os.environ["CIMEROOT"] = cimeroot - os.environ["CASETOOLS"] = casetools - os.environ["EXEROOT"] = exeroot - os.environ["INCROOT"] = incroot - os.environ["LIBROOT"] = libroot - os.environ["SHAREDLIBROOT"] = sharedlibroot - os.environ["CASEROOT"] = caseroot - os.environ["COMPILER"] = compiler - os.environ["COMP_INTERFACE"] = comp_interface - os.environ["NINST_VALUE"] = str(ninst_value) - os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) - os.environ["MACH"] = mach - os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) - os.environ["MPILIB"] = mpilib - os.environ["DEBUG"] = stringify_bool(debug) - os.environ["OS"] = os_ - os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" - os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else "" - os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else "" - os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else "" - os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable) - os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc) - os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos) - os.environ["MPASLI_USE_ALBANY"] = stringify_bool(mpasli_use_albany) - - if get_model() == "acme" and mach == "titan" and compiler == "pgiacc": - case.set_value("CAM_TARGET", "preqx_acc") - - # This is a timestamp for the build , not the same as the testid, - # and this case may not be a test anyway. For a production - # experiment there may be many builds of the same case. - lid = run_cmd_no_fail("date +%y%m%d-%H%M%S") - os.environ["LID"] = lid - - # Set the overall USE_PETSC variable to TRUE if any of the - # *_USE_PETSC variables are TRUE. - # For now, there is just the one CLM_USE_PETSC variable, but in - # the future there may be others -- so USE_PETSC will be true if - # ANY of those are true. - - use_petsc = clm_use_petsc - case.set_value("USE_PETSC", use_petsc) - os.environ["USE_PETSC"] = stringify_bool(use_petsc) - - # Set the overall USE_TRILINOS variable to TRUE if any of the - # *_USE_TRILINOS variables are TRUE. - # For now, there is just the one CISM_USE_TRILINOS variable, but in - # the future there may be others -- so USE_TRILINOS will be true if - # ANY of those are true. - - use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos - case.set_value("USE_TRILINOS", use_trilinos) - os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos) - - # Set the overall USE_ALBANY variable to TRUE if any of the - # *_USE_ALBANY variables are TRUE. - # For now, there is just the one MPASLI_USE_ALBANY variable, but in - # the future there may be others -- so USE_ALBANY will be true if - # ANY of those are true. - - use_albany = stringify_bool(mpasli_use_albany) - case.set_value("USE_ALBANY", use_albany) - os.environ["USE_ALBANY"] = use_albany - - # Load modules - case.load_env() - - sharedpath = build_checks(case, build_threaded, comp_interface, - use_esmf_lib, debug, compiler, mpilib, - complist, ninst_build, smp_value, model_only) - - t2 = time.time() - logs = [] - - if not model_only: - logs = build_libraries(case, exeroot, sharedpath, caseroot, - cimeroot, libroot, lid, compiler) - - if not sharedlib_only: - os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators - logs.extend(build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, - lid, caseroot, cimeroot, compiler)) - - if not sharedlib_only: - # in case component build scripts updated the xml files, update the case object - case.read_xml() - post_build(case, logs) - - t3 = time.time() - - logger.info("Time spent not building: %f sec" % (t2 - t1)) - logger.info("Time spent building: %f sec" % (t3 - t2)) - - return True - -############################################################################### -def build_checks(case, build_threaded, comp_interface, use_esmf_lib, - debug, compiler, mpilib, complist, ninst_build, smp_value, - model_only): +def _build_checks(case, build_threaded, comp_interface, use_esmf_lib, + debug, compiler, mpilib, complist, ninst_build, smp_value, + model_only): ############################################################################### """ check if a build needs to be done and warn if a clean is warrented first @@ -408,6 +211,9 @@ def build_checks(case, build_threaded, comp_interface, use_esmf_lib, case.set_value("BUILD_COMPLETE", False) + # User may have rm -rf their build directory + create_dirs(case) + case.flush() if not model_only: logger.info("Generating component namelists as part of build") @@ -416,7 +222,7 @@ def build_checks(case, build_threaded, comp_interface, use_esmf_lib, return sharedpath ############################################################################### -def build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler): +def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler): ############################################################################### shared_lib = os.path.join(exeroot, sharedpath, "lib") @@ -443,15 +249,14 @@ def build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, os.makedirs(full_lib_path) file_build = os.path.join(exeroot, "%s.bldlog.%s" % (lib, lid)) - my_file = os.path.join(cimeroot, "cime_config", "buildlib.%s" % lib) + my_file = os.path.join(cimeroot, "src", "build_scripts", "buildlib.%s" % lib) if lib == "pio": my_file = "PYTHONPATH=%s:%s:$PYTHONPATH %s"%(os.path.join(cimeroot,"scripts","Tools"), - os.path.join(cimeroot,"utils","python"), my_file) + os.path.join(cimeroot,"scripts","lib"), my_file) logger.info("Building %s with output to file %s"%(lib,file_build)) - with open(file_build, "w") as fd: - stat = run_cmd("%s %s %s %s 2>&1" % - (my_file, full_lib_path, os.path.join(exeroot,sharedpath), caseroot), - from_dir=exeroot,arg_stdout=fd)[0] + stat = run_cmd("%s %s %s %s" % + (my_file, full_lib_path, os.path.join(exeroot,sharedpath), caseroot), + from_dir=exeroot, combine_output=True, arg_stdout=file_build)[0] analyze_build_log(lib, file_build, compiler) expect(stat == 0, "ERROR: buildlib.%s failed, cat %s" % (lib, file_build)) @@ -510,7 +315,7 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro ############################################################################### -def clean(case, cleanlist=None, clean_all=False): +def _clean_impl(case, cleanlist, clean_all): ############################################################################### caseroot = case.get_value("CASEROOT") if clean_all: @@ -530,7 +335,7 @@ def clean(case, cleanlist=None, clean_all=False): expect(cleanlist is not None and len(cleanlist) > 0,"Empty cleanlist not expected") debug = case.get_value("DEBUG") use_esmf_lib = case.get_value("USE_ESMF_LIB") - build_threaded = case.get_value("BUILD_THREADED") + build_threaded = case.get_build_threaded() gmake = case.get_value("GMAKE") caseroot = case.get_value("CASEROOT") casetools = case.get_value("CASETOOLS") @@ -560,10 +365,211 @@ def clean(case, cleanlist=None, clean_all=False): case.set_value("BUILD_COMPLETE","FALSE") case.flush() - # append call of to CaseStatus - if cleanlist: - msg = "clean %s "%" ".join(cleanlist) - elif clean_all: - msg = "clean_all" +############################################################################### +def _case_build_impl(caseroot, case, sharedlib_only, model_only): +############################################################################### - append_status(msg, caseroot=caseroot, sfile="CaseStatus") + t1 = time.time() + + expect(not (sharedlib_only and model_only), + "Contradiction: both sharedlib_only and model_only") + logger.info("Building case in directory %s"%caseroot) + logger.info("sharedlib_only is %s" % sharedlib_only) + logger.info("model_only is %s" % model_only) + + expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot) + os.chdir(caseroot) + + expect(os.path.exists("case.run"), + "ERROR: must invoke case.setup script before calling build script ") + + cimeroot = case.get_value("CIMEROOT") + + comp_classes = case.get_values("COMP_CLASSES") + + check_lockedfiles(caseroot) + + # Retrieve relevant case data + # This environment variable gets set for cesm Make and + # needs to be unset before building again. + if "MODEL" in os.environ.keys(): + del os.environ["MODEL"] + build_threaded = case.get_build_threaded() + casetools = case.get_value("CASETOOLS") + exeroot = case.get_value("EXEROOT") + incroot = case.get_value("INCROOT") + libroot = case.get_value("LIBROOT") + sharedlibroot = case.get_value("SHAREDLIBROOT") + + complist = [] + for comp_class in comp_classes: + if comp_class == "CPL": + ninst = 1 + config_dir = None + else: + ninst = case.get_value("NINST_%s"%comp_class) + config_dir = os.path.dirname(case.get_value("CONFIG_%s_FILE"%comp_class)) + comp = case.get_value("COMP_%s"%comp_class) + thrds = case.get_value("NTHRDS_%s"%comp_class) + expect(ninst is not None,"Failed to get ninst for comp_class %s"%comp_class) + complist.append((comp_class.lower(), comp, thrds, ninst, config_dir )) + os.environ["COMP_%s"%comp_class] = comp + + ocn_submodel = case.get_value("OCN_SUBMODEL") + profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE") + compiler = case.get_value("COMPILER") + comp_interface = case.get_value("COMP_INTERFACE") + mpilib = case.get_value("MPILIB") + use_esmf_lib = case.get_value("USE_ESMF_LIB") + debug = case.get_value("DEBUG") + ninst_build = case.get_value("NINST_BUILD") + smp_value = case.get_value("SMP_VALUE") + clm_use_petsc = case.get_value("CLM_USE_PETSC") + cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") + mpasli_use_albany = case.get_value("MPASLI_USE_ALBANY") + clm_config_opts = case.get_value("CLM_CONFIG_OPTS") + cam_config_opts = case.get_value("CAM_CONFIG_OPTS") + pio_config_opts = case.get_value("PIO_CONFIG_OPTS") + ninst_value = case.get_value("NINST_VALUE") + mach = case.get_value("MACH") + os_ = case.get_value("OS") + # Load some params into env + os.environ["CIMEROOT"] = cimeroot + os.environ["CASETOOLS"] = casetools + os.environ["EXEROOT"] = exeroot + os.environ["INCROOT"] = incroot + os.environ["LIBROOT"] = libroot + os.environ["SHAREDLIBROOT"] = sharedlibroot + os.environ["CASEROOT"] = caseroot + os.environ["COMPILER"] = compiler + os.environ["COMP_INTERFACE"] = comp_interface + os.environ["NINST_VALUE"] = str(ninst_value) + os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) + os.environ["MACH"] = mach + os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) + os.environ["MPILIB"] = mpilib + os.environ["DEBUG"] = stringify_bool(debug) + os.environ["OS"] = os_ + os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" + os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else "" + os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else "" + os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else "" + os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable) + os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc) + os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos) + os.environ["MPASLI_USE_ALBANY"] = stringify_bool(mpasli_use_albany) + + if get_model() == "acme" and mach == "titan" and compiler == "pgiacc": + case.set_value("CAM_TARGET", "preqx_acc") + + # This is a timestamp for the build , not the same as the testid, + # and this case may not be a test anyway. For a production + # experiment there may be many builds of the same case. + lid = run_cmd_no_fail("date +%y%m%d-%H%M%S") + os.environ["LID"] = lid + + # Set the overall USE_PETSC variable to TRUE if any of the + # *_USE_PETSC variables are TRUE. + # For now, there is just the one CLM_USE_PETSC variable, but in + # the future there may be others -- so USE_PETSC will be true if + # ANY of those are true. + + use_petsc = clm_use_petsc + case.set_value("USE_PETSC", use_petsc) + os.environ["USE_PETSC"] = stringify_bool(use_petsc) + + # Set the overall USE_TRILINOS variable to TRUE if any of the + # *_USE_TRILINOS variables are TRUE. + # For now, there is just the one CISM_USE_TRILINOS variable, but in + # the future there may be others -- so USE_TRILINOS will be true if + # ANY of those are true. + + use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos + case.set_value("USE_TRILINOS", use_trilinos) + os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos) + + # Set the overall USE_ALBANY variable to TRUE if any of the + # *_USE_ALBANY variables are TRUE. + # For now, there is just the one MPASLI_USE_ALBANY variable, but in + # the future there may be others -- so USE_ALBANY will be true if + # ANY of those are true. + + use_albany = stringify_bool(mpasli_use_albany) + case.set_value("USE_ALBANY", use_albany) + os.environ["USE_ALBANY"] = use_albany + + # Load modules + case.load_env() + + sharedpath = _build_checks(case, build_threaded, comp_interface, + use_esmf_lib, debug, compiler, mpilib, + complist, ninst_build, smp_value, model_only) + + t2 = time.time() + logs = [] + + if not model_only: + logs = _build_libraries(case, exeroot, sharedpath, caseroot, + cimeroot, libroot, lid, compiler) + + if not sharedlib_only: + os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators + logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, + lid, caseroot, cimeroot, compiler)) + + if not sharedlib_only: + # in case component build scripts updated the xml files, update the case object + case.read_xml() + post_build(case, logs) + + t3 = time.time() + + logger.info("Time spent not building: %f sec" % (t2 - t1)) + logger.info("Time spent building: %f sec" % (t3 - t2)) + + return True + +############################################################################### +def post_build(case, logs): +############################################################################### + + logdir = case.get_value("LOGDIR") + + #zip build logs to CASEROOT/logs + if logdir: + bldlogdir = os.path.join(logdir, "bld") + if not os.path.exists(bldlogdir): + os.makedirs(bldlogdir) + + for log in logs: + logger.debug("Copying build log %s to %s"%(log,bldlogdir)) + with open(log, 'rb') as f_in: + with gzip.open("%s.gz"%log, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + if "sharedlibroot" not in log: + shutil.copy("%s.gz"%log,os.path.join(bldlogdir,"%s.gz"%os.path.basename(log))) + + # Set XML to indicate build complete + case.set_value("BUILD_COMPLETE", True) + case.set_value("BUILD_STATUS", 0) + if "SMP_VALUE" in os.environ: + case.set_value("SMP_BUILD", os.environ["SMP_VALUE"]) + case.flush() + + lock_file("env_build.xml") + + # must ensure there's an lid + lid = os.environ["LID"] if "LID" in os.environ else run_cmd_no_fail("date +%y%m%d-%H%M%S") + save_build_provenance(case, lid=lid) + +############################################################################### +def case_build(caseroot, case, sharedlib_only=False, model_only=False): +############################################################################### + functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only) + return run_and_log_case_status(functor, "case.build", caseroot=caseroot) + +############################################################################### +def clean(case, cleanlist=None, clean_all=False): +############################################################################### + functor = lambda: _clean_impl(case, cleanlist, clean_all) + return run_and_log_case_status(functor, "build.clean", caseroot=case.get_value("CASEROOT")) diff --git a/utils/python/CIME/buildlib.py b/scripts/lib/CIME/buildlib.py similarity index 89% rename from utils/python/CIME/buildlib.py rename to scripts/lib/CIME/buildlib.py index 55d269961ec..33b4ffce959 100644 --- a/utils/python/CIME/buildlib.py +++ b/scripts/lib/CIME/buildlib.py @@ -50,7 +50,7 @@ def build_data_lib(argv, compclass): compname = "d" + compclass with open('Filepath', 'w') as out: out.write(os.path.join(caseroot, "SourceMods", "src.%s" %compname) + "\n") - out.write(os.path.join(cimeroot, "components", "data_comps", compname) + "\n") + out.write(os.path.join(cimeroot, "src", "components", "data_comps", compname) + "\n") # Build the component run_gmake(case, compclass, libroot) @@ -69,8 +69,8 @@ def build_xcpl_lib(argv, compclass): compname = "x" + compclass with open('Filepath', 'w') as out: out.write(os.path.join(caseroot, "SourceMods", "src.%s", compname) + "\n") - out.write(os.path.join(cimeroot, "components", "xcpl_comps", "xshare") + "\n") - out.write(os.path.join(cimeroot, "components", "xcpl_comps",compname, "cpl") + "\n") + out.write(os.path.join(cimeroot, "src", "components", "xcpl_comps", "xshare") + "\n") + out.write(os.path.join(cimeroot, "src", "components", "xcpl_comps",compname, "cpl") + "\n") # Build the component run_gmake(case, compclass, libroot) @@ -89,8 +89,8 @@ def build_stub_lib(argv, compclass): compname = "s" + compclass with open('Filepath', 'w') as out: out.write(os.path.join(caseroot, "SourceMods", "src.%s", compname) + "\n") - out.write(os.path.join(cimeroot, "components", "stub_comps", "xshare") + "\n") - out.write(os.path.join(cimeroot, "components", "stub_comps",compname, "cpl") + "\n") + out.write(os.path.join(cimeroot, "src", "components", "stub_comps", "xshare") + "\n") + out.write(os.path.join(cimeroot, "src", "components", "stub_comps",compname, "cpl") + "\n") # Build the component run_gmake(case, compclass, libroot) diff --git a/utils/python/CIME/buildnml.py b/scripts/lib/CIME/buildnml.py similarity index 99% rename from utils/python/CIME/buildnml.py rename to scripts/lib/CIME/buildnml.py index 2d90636a7b1..34f863155b6 100644 --- a/utils/python/CIME/buildnml.py +++ b/scripts/lib/CIME/buildnml.py @@ -118,6 +118,5 @@ def create_namelist_infile(case, user_nl_file, namelist_infile, infile_text=""): lines_output.append(line) lines_output.append("/ \n") - with open(namelist_infile, "w") as file_infile: file_infile.write("\n".join(lines_output)) diff --git a/utils/python/CIME/case.py b/scripts/lib/CIME/case.py similarity index 94% rename from utils/python/CIME/case.py rename to scripts/lib/CIME/case.py index 33d589e50a6..01cd66de356 100644 --- a/utils/python/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -4,25 +4,24 @@ All interaction with and between the module files in XML/ takes place through the Case module. """ -from copy import deepcopy +from copy import deepcopy import glob, os, shutil, math, string from CIME.XML.standard_module_setup import * from CIME.utils import expect, get_cime_root, append_status from CIME.utils import convert_to_type, get_model, get_project -from CIME.utils import get_build_threaded, get_current_commit +from CIME.utils import get_current_commit from CIME.check_lockedfiles import LOCKED_DIR, lock_file from CIME.XML.machines import Machines from CIME.XML.pes import Pes from CIME.XML.files import Files -from CIME.XML.testlist import Testlist +from CIME.XML.testlist import Testlist from CIME.XML.component import Component from CIME.XML.compsets import Compsets from CIME.XML.grids import Grids from CIME.XML.batch import Batch from CIME.XML.pio import PIO from CIME.XML.archive import Archive - from CIME.XML.env_test import EnvTest from CIME.XML.env_mach_specific import EnvMachSpecific from CIME.XML.env_case import EnvCase @@ -97,18 +96,17 @@ def __init__(self, case_root=None, read_only=True): self._component_classes = [] self._is_env_loaded = False - self.thread_count = None self.total_tasks = None self.tasks_per_node = None self.num_nodes = None + self.spare_nodes = None self.tasks_per_numa = None self.cores_per_task = None # check if case has been configured and if so initialize derived if self.get_value("CASEROOT") is not None: self._initialize_derived_attributes() - def check_if_comp_var(self, vid): vid = vid comp = None @@ -124,9 +122,10 @@ def _initialize_derived_attributes(self): These are derived variables which can be used in the config_* files for variable substitution using the {{ var }} syntax """ - env_mach_pes = self.get_env("mach_pes") - comp_classes = self.get_values("COMP_CLASSES") - pes_per_node = self.get_value("PES_PER_NODE") + env_mach_pes = self.get_env("mach_pes") + env_mach_spec = self.get_env('mach_specific') + comp_classes = self.get_values("COMP_CLASSES") + pes_per_node = self.get_value("PES_PER_NODE") self.total_tasks = env_mach_pes.get_total_tasks(comp_classes) self.thread_count = env_mach_pes.get_max_thread_count(comp_classes) @@ -140,10 +139,21 @@ def _initialize_derived_attributes(self): threads_per_core = 1 if (threads_per_node <= pes_per_node) else smt_factor self.cores_per_task = self.thread_count / threads_per_core - if self.get_value("MACH") == "titan": + mpi_attribs = { + "compiler" : self.get_value("COMPILER"), + "mpilib" : self.get_value("MPILIB"), + "threaded" : self.get_build_threaded(), + "unit_testing" : False + } + + executable = env_mach_spec.get_mpirun(self, mpi_attribs, job="case.run", exe_only=True)[0] + if executable == "aprun": self.num_nodes = get_aprun_cmd_for_case(self, "acme.exe")[1] + self.spare_nodes = env_mach_pes.get_spare_nodes(self.num_nodes) + self.num_nodes += self.spare_nodes else: - self.num_nodes = env_mach_pes.get_total_nodes(self.total_tasks, self.thread_count) + self.num_nodes, self.spare_nodes = env_mach_pes.get_total_nodes(self.total_tasks, self.thread_count) + self.num_nodes += self.spare_nodes # Define __enter__ and __exit__ so that we can use this as a context manager # and force a flush on exit. @@ -176,10 +186,10 @@ def read_xml(self): self._env_entryid_files.append(EnvRun(self._caseroot, components=components)) self._env_entryid_files.append(EnvBuild(self._caseroot, components=components)) self._env_entryid_files.append(EnvMachPes(self._caseroot, components=components)) + self._env_entryid_files.append(EnvBatch(self._caseroot)) if os.path.isfile(os.path.join(self._caseroot,"env_test.xml")): self._env_entryid_files.append(EnvTest(self._caseroot, components=components)) self._env_generic_files = [] - self._env_generic_files.append(EnvBatch(self._caseroot)) self._env_generic_files.append(EnvMachSpecific(self._caseroot)) self._env_generic_files.append(EnvArchive(self._caseroot)) self._files = self._env_entryid_files + self._env_generic_files @@ -284,12 +294,7 @@ def get_value(self, item, attribute=None, resolved=True, subgroup=None): # Return empty result return result - def get_record_fields(self, variable, field): - - """ - - """ # Empty result result = [] @@ -338,8 +343,8 @@ def get_type_info(self, item): result = env_file.get_type_info(item) if result is not None: return result - env_batch = self.get_env("batch") - return env_batch.get_type_info(item) + + return result def get_resolved_value(self, item, recurse=0): num_unresolved = item.count("$") if item else 0 @@ -350,22 +355,7 @@ def get_resolved_value(self, item, recurse=0): if ("$" not in item): return item else: - item = self.get_resolved_value(item,recurse=recurse+1) - - if recurse >= 2*recurse_limit: - logging.warning("Not able to fully resolve item '%s'" % item) - elif recurse >= recurse_limit: - #try env_batch first - env_batch = self.get_env("batch") - item = env_batch.get_resolved_value(item) - logger.debug("item is %s, checking env_batch"%item) - if item is not None: - if ("$" not in item): - return item - else: - item = self.get_resolved_value(item,recurse=recurse+1) - else: - logging.warning("Not able to fully resolve item '%s'" % item) + item = self.get_resolved_value(item, recurse=recurse+1) return item @@ -378,9 +368,7 @@ def set_value(self, item, value, subgroup=None, ignore_type=False): if item == "CASEROOT": self._caseroot = value result = None - files = self._env_entryid_files - files.append(self.get_env('batch')) - for env_file in files: + for env_file in self._env_entryid_files: result = env_file.set_value(item, value, subgroup, ignore_type) if (result is not None): logger.debug("Will rewrite file %s %s",env_file.filename, item) @@ -416,7 +404,7 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf either a longname or an alias. This will also set the compsets and pes specfication files. """ - science_support = {} + science_support = [] compset_alias = None components = files.get_components("COMPSETS_SPEC_FILE") logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components) @@ -460,7 +448,8 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf self.set_lookup_value("PES_SPEC_FILE", pesfile) else: expect(False, - "Could not find a compset match for either alias or longname in %s" %(compset_name)) + "Could not find a compset match for either alias or longname in %s\n"%(compset_name) + + "You may need the --user-compset argument.") return None, science_support @@ -521,10 +510,6 @@ def _get_component_config_data(self, files): for env_file in self._env_entryid_files: env_file.add_elements_by_group(drv_comp_model_specific, attributes=attlist) - # Add the group and elements for env_batch - env_batch = self.get_env("batch") - env_batch.add_elements_by_group(drv_comp, attributes=attlist) - # loop over all elements of both component_classes and components - and get config_component_file for # for each component self._set_comp_classes(drv_comp.get_valid_model_components()) @@ -562,7 +547,7 @@ def configure(self, compset_name, grid_name, machine_name=None, project=None, pecount=None, compiler=None, mpilib=None, user_compset=False, pesfile=None, user_grid=False, gridfile=None, ninst=1, test=False, - walltime=None, queue=None, output_root=None, run_unsupported=False): + walltime=None, queue=None, output_root=None, run_unsupported=False, answer=None): #-------------------------------------------- # compset, pesfile, and compset components @@ -684,7 +669,6 @@ def configure(self, compset_name, grid_name, machine_name=None, mach_pes_obj = self.get_env("mach_pes") - if other is not None: for key, value in other.items(): self.set_value(key, value) @@ -734,7 +718,6 @@ def configure(self, compset_name, grid_name, machine_name=None, batch = Batch(batch_system=batch_system_type, machine=machine_name) bjobs = batch.get_batch_jobs() - env_batch.set_batch_system(batch, batch_system_type=batch_system_type) env_batch.create_job_groups(bjobs) env_batch.set_job_defaults(bjobs, pesize=maxval, walltime=walltime, force_queue=queue) @@ -765,8 +748,6 @@ def configure(self, compset_name, grid_name, machine_name=None, else: self._check_testlists(compset_alias, grid_name, files) - - # Set project id if project is None: project = get_project(machobj) @@ -788,7 +769,11 @@ def configure(self, compset_name, grid_name, machine_name=None, logging.debug("wdir is %s"%wdir) if os.path.exists(wdir): expect(not test, "Directory %s already exists, aborting test"% wdir) - response = raw_input("\nDirectory %s already exists, (r)eplace, (a)bort, or (u)se existing?"% wdir) + if answer is None: + response = raw_input("\nDirectory %s already exists, (r)eplace, (a)bort, or (u)se existing?"% wdir) + else: + response = answer + if response.startswith("r"): shutil.rmtree(wdir) else: @@ -900,12 +885,6 @@ def _create_caseroot_tools(self): else: shutil.copy(os.path.join(machines_dir, "syslog.noop"), os.path.join(casetools, "mach_syslog")) - if get_model() == "acme": - if os.path.exists(os.path.join(machines_dir, "syslog.%s" % machine)): - shutil.copy(os.path.join(machines_dir, "syslog.%s" % machine), os.path.join(casetools, "mach_syslog")) - else: - shutil.copy(os.path.join(machines_dir, "syslog.noop"), os.path.join(casetools, "mach_syslog")) - def _create_caseroot_sourcemods(self): components = self.get_compset_components() for component in components: @@ -953,21 +932,21 @@ def create_caseroot(self, clone=False): os.makedirs(newdir) # Open a new README.case file in $self._caseroot - append_status(" ".join(sys.argv), caseroot=self._caseroot, sfile="README.case") + append_status(" ".join(sys.argv), "README.case", caseroot=self._caseroot) append_status("Compset longname is %s"%self.get_value("COMPSET"), - caseroot=self._caseroot, sfile="README.case") + "README.case", caseroot=self._caseroot) append_status("Compset specification file is %s" % (self.get_value("COMPSETS_SPEC_FILE")), - caseroot=self._caseroot, sfile="README.case") + "README.case", caseroot=self._caseroot) append_status("Pes specification file is %s" % (self.get_value("PES_SPEC_FILE")), - caseroot=self._caseroot, sfile="README.case") + "README.case", caseroot=self._caseroot) for component_class in self._component_classes: if component_class == "CPL": continue comp_grid = "%s_GRID"%component_class append_status("%s is %s"%(comp_grid,self.get_value(comp_grid)), - caseroot=self._caseroot, sfile="README.case") + "README.case", caseroot=self._caseroot) if not clone: self._create_caseroot_sourcemods() self._create_caseroot_tools() @@ -1029,6 +1008,10 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime orig_exeroot = self.get_value("EXEROOT") newcase.set_value("EXEROOT", orig_exeroot) newcase.set_value("BUILD_COMPLETE","TRUE") + orig_bld_complete = self.get_value("BUILD_COMPLETE") + if not orig_bld_complete: + logger.warn("\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone") + logger.warn("Avoid this message by building case one before you clone.\n") else: newcase.set_value("BUILD_COMPLETE","FALSE") @@ -1058,8 +1041,10 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime shutil.copy(item, newcaseroot) # copy SourceMod and Buildconf files + # if symlinks exist, copy rather than follow links for casesub in ("SourceMods", "Buildconf"): - shutil.copytree(os.path.join(cloneroot, casesub), os.path.join(newcaseroot, casesub)) + shutil.copytree(os.path.join(cloneroot, casesub), os.path.join(newcaseroot, casesub) + , symlinks=True) # lock env_case.xml in new case lock_file("env_case.xml", newcaseroot) @@ -1073,7 +1058,7 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime clonename = self.get_value("CASE") logger.info(" Successfully created new case %s from clone case %s " %(newcasename, clonename)) - case_setup(newcase, clean=False, test_mode=False) + case_setup(newcase) return newcase @@ -1092,14 +1077,17 @@ def get_mpirun_cmd(self, job="case.run"): mpi_attribs = { "compiler" : self.get_value("COMPILER"), "mpilib" : self.get_value("MPILIB"), - "threaded" : get_build_threaded(self) + "threaded" : self.get_build_threaded(), + "unit_testing" : False } executable, args = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) # special case for aprun if executable == "aprun": - return get_aprun_cmd_for_case(self, run_exe)[0] + " " + run_misc_suffix + aprun_cmd, num_nodes = get_aprun_cmd_for_case(self, run_exe) + expect( (num_nodes + self.spare_nodes) == self.num_nodes, "Not using optimized num nodes") + return aprun_cmd + " " + run_misc_suffix else: mpi_arg_string = " ".join(args.values()) @@ -1137,6 +1125,13 @@ def load_env(self): env_module.load_env(compiler=compiler,debug=debug, mpilib=mpilib) self._is_env_loaded = True + def get_build_threaded(self): + """ + Returns True if current settings require a threaded build/run. + """ + force_threaded = self.get_value("BUILD_THREADED") + return bool(force_threaded) or self.thread_count > 1 + def _check_testlists(self, compset_alias, grid_name, files): """ CESM only: check the testlist file for tests of this compset grid combination diff --git a/utils/python/CIME/case_cmpgen_namelists.py b/scripts/lib/CIME/case_cmpgen_namelists.py similarity index 98% rename from utils/python/CIME/case_cmpgen_namelists.py rename to scripts/lib/CIME/case_cmpgen_namelists.py index 2ca8d964fe3..f1482d3e532 100644 --- a/utils/python/CIME/case_cmpgen_namelists.py +++ b/scripts/lib/CIME/case_cmpgen_namelists.py @@ -129,7 +129,7 @@ def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None logging.warning(warn) finally: ts.set_status(NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS) - append_status(output, caseroot=caseroot, sfile=logfile_name) + append_status(output, logfile_name, caseroot=caseroot) return success diff --git a/utils/python/CIME/case_lt_archive.py b/scripts/lib/CIME/case_lt_archive.py similarity index 79% rename from utils/python/CIME/case_lt_archive.py rename to scripts/lib/CIME/case_lt_archive.py index e3c69bd78fb..51f2d9d9655 100644 --- a/utils/python/CIME/case_lt_archive.py +++ b/scripts/lib/CIME/case_lt_archive.py @@ -1,5 +1,5 @@ from CIME.XML.standard_module_setup import * -from CIME.utils import expect, does_file_have_string, append_status +from CIME.utils import expect, does_file_have_string, run_and_log_case_status from CIME.XML.lt_archive import LTArchive import time @@ -7,16 +7,13 @@ logger = logging.getLogger(__name__) ############################################################################### -def case_lt_archive(case): +def _case_lt_archive_impl(case): ############################################################################### caseroot = case.get_value("CASEROOT") # max number of threads needed by scripts os.environ["maxthrds"] = "1" - # document start - append_status("lt_archive starting",caseroot=caseroot,sfile="CaseStatus") - # determine status of run and short term archiving runComplete = does_file_have_string(os.path.join(caseroot, "CaseStatus"), "Run SUCCESSFUL") @@ -37,12 +34,15 @@ def case_lt_archive(case): + lt_archive_args + "ltArchiveStatus." + lid + " 2>&1" run_cmd_no_fail(cmd, from_dir=caseroot) else: - logger.warn("runComplete %s staComplete %s"%(runComplete, staComplete)) + logger.warn("runComplete %s staComplete %s" % (runComplete, staComplete)) expect(False, "lt_archive: run or st_archive is not yet complete or was not successful." "Unable to perform long term archive...") - # document completion - append_status("lt_archive completed" ,caseroot=caseroot, sfile="CaseStatus") - return True + +############################################################################### +def case_lt_archive(case): +############################################################################### + functor = lambda: _case_lt_archive_impl(case) + return run_and_log_case_status(functor, "lt_archiving", caseroot=case.get_value("CASEROOT")) diff --git a/utils/python/CIME/case_run.py b/scripts/lib/CIME/case_run.py similarity index 65% rename from utils/python/CIME/case_run.py rename to scripts/lib/CIME/case_run.py index 0d4224fd873..0b4728b2f46 100644 --- a/utils/python/CIME/case_run.py +++ b/scripts/lib/CIME/case_run.py @@ -1,12 +1,13 @@ from CIME.XML.standard_module_setup import * from CIME.case_submit import submit -from CIME.utils import append_status, gzip_existing_file, new_lid +from CIME.utils import gzip_existing_file, new_lid, run_and_log_case_status from CIME.check_lockedfiles import check_lockedfiles from CIME.get_timing import get_timing from CIME.provenance import save_prerun_provenance, save_postrun_provenance from CIME.preview_namelists import create_namelists +from CIME.case_st_archive import case_st_archive, restore_from_archive -import shutil, time, sys, os, glob, resource +import shutil, time, sys, os, glob logger = logging.getLogger(__name__) @@ -26,7 +27,6 @@ def pre_run_check(case, lid): env_mach_pes = os.path.join(caseroot,"env_mach_pes.xml") shutil.copy(env_mach_pes,"%s.%s"%(env_mach_pes,lid)) - # check for locked files. check_lockedfiles(case.get_value("CASEROOT")) logger.debug("check_lockedfiles OK") @@ -69,10 +69,6 @@ def pre_run_check(case, lid): # variable while the job is in the queue create_namelists(case) - # document process - append_status("Run started ", caseroot=caseroot, - sfile="CaseStatus") - logger.info("-------------------------------------------------------------------------") logger.info(" - Prestage required restarts into %s" %(rundir)) logger.info(" - Case input data directory (DIN_LOC_ROOT) is %s " %(din_loc_root)) @@ -80,9 +76,19 @@ def pre_run_check(case, lid): logger.info("-------------------------------------------------------------------------") ############################################################################### -def run_model(case): +def _run_model_impl(case, lid): ############################################################################### + pre_run_check(case, lid) + + model = case.get_value("MODEL") + + # Set OMP_NUM_THREADS + env_mach_pes = case.get_env("mach_pes") + comp_classes = case.get_values("COMP_CLASSES") + thread_count = env_mach_pes.get_max_thread_count(comp_classes) + os.environ["OMP_NUM_THREADS"] = str(thread_count) + # Run the model logger.info("%s MODEL EXECUTION BEGINS HERE" %(time.strftime("%Y-%m-%d %H:%M:%S"))) @@ -91,14 +97,58 @@ def run_model(case): logger.info("run command is %s " %cmd) rundir = case.get_value("RUNDIR") - run_cmd_no_fail(cmd, from_dir=rundir) + loop = True + + while loop: + loop = False + stat = run_cmd(cmd, from_dir=rundir)[0] + model_logfile = os.path.join(rundir, model + ".log." + lid) + # Determine if failure was due to a failed node, if so, try to restart + if stat != 0: + node_fail_re = case.get_value("NODE_FAIL_REGEX") + if node_fail_re: + node_fail_regex = re.compile(node_fail_re) + model_logfile = os.path.join(rundir, model + ".log." + lid) + if os.path.exists(model_logfile): + num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read())) + if num_fails > 0 and case.spare_nodes >= num_fails: + # We failed due to node failure! + logger.warning("Detected model run failed due to node failure, restarting") + + # Archive the last consistent set of restart files and restore them + case_st_archive(case, no_resubmit=True) + restore_from_archive(case) + + orig_cont = case.get_value("CONTINUE_RUN") + if not orig_cont: + case.set_value("CONTINUE_RUN", True) + create_namelists(case) + + lid = new_lid() + loop = True + + case.spare_nodes -= num_fails + + if not loop: + # We failed and we're not restarting + expect(False, "RUN FAIL: Command '%s' failed\nSee log file for details: %s" % (cmd, model_logfile)) + logger.info("%s MODEL EXECUTION HAS FINISHED" %(time.strftime("%Y-%m-%d %H:%M:%S"))) + post_run_check(case, lid) + + return lid + +############################################################################### +def run_model(case, lid): +############################################################################### + functor = lambda: _run_model_impl(case, lid) + return run_and_log_case_status(functor, "case.run", caseroot=case.get_value("CASEROOT")) + ############################################################################### def post_run_check(case, lid): ############################################################################### - caseroot = case.get_value("CASEROOT") rundir = case.get_value("RUNDIR") model = case.get_value("MODEL") @@ -107,26 +157,15 @@ def post_run_check(case, lid): cpl_logfile = os.path.join(rundir, "cpl" + ".log." + lid) if not os.path.isfile(model_logfile): - msg = "Model did not complete, no %s log file "%model_logfile - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - expect(False, msg) + expect(False, "Model did not complete, no %s log file " % model_logfile) elif not os.path.isfile(cpl_logfile): - msg = "Model did not complete, no cpl log file" - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - expect(False, msg) + expect(False, "Model did not complete, no cpl log file '%s'" % cpl_logfile) elif os.stat(model_logfile).st_size == 0: - msg = " Run FAILED " - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - expect(False, msg) + expect(False, "Run FAILED") else: with open(cpl_logfile, 'r') as fd: - if 'SUCCESSFUL TERMINATION' in fd.read(): - msg = "Run SUCCESSFUL" - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - else: - msg = "Model did not complete - see %s \n " %(cpl_logfile) - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - expect(False, msg) + if 'SUCCESSFUL TERMINATION' not in fd.read(): + expect(False, "Model did not complete - see %s \n " % cpl_logfile) ############################################################################### def save_logs(case, lid): @@ -178,12 +217,22 @@ def resubmit_check(case): submit(case, job=job, resubmit=True) ############################################################################### -def do_data_assimilation(da_script, caseroot, cycle, lid): +def do_external(script_name, caseroot, rundir, lid, prefix): ############################################################################### - cmd = da_script + " 1> da.log.%s %s %d 2>&1" %(lid, caseroot, cycle) - logger.debug("running %s" %da_script) + filename = "%s.external.log.%s" %(prefix, lid) + outfile = os.path.join(rundir, filename) + cmd = script_name + " 1> %s %s 2>&1" %(outfile, caseroot) + logger.info("running %s" %script_name) + run_cmd_no_fail(cmd) + +############################################################################### +def do_data_assimilation(da_script, caseroot, cycle, lid, rundir): +############################################################################### + filename = "da.log.%s" %(lid) + outfile = os.path.join(rundir, filename) + cmd = da_script + " 1> %s %s %d 2>&1" %(outfile, caseroot, cycle) + logger.info("running %s" %da_script) run_cmd_no_fail(cmd) - # disposeLog(case, 'da', lid) THIS IS UNDEFINED! ############################################################################### def case_run(case): @@ -196,12 +245,8 @@ def case_run(case): "Please submit your run using the submit script like so:" " ./case.submit") - # If we need to produce core files, we need to try to ensure that the - # system will support them - try: - resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) - except: - pass + prerun_script = case.get_value("PRERUN_SCRIPT") + postrun_script = case.get_value("POSTRUN_SCRIPT") data_assimilation = case.get_value("DATA_ASSIMILATION") data_assimilation_cycles = case.get_value("DATA_ASSIMILATION_CYCLES") @@ -210,11 +255,6 @@ def case_run(case): # set up the LID lid = new_lid() - # Set OMP_NUM_THREADS. Want this to happen before prerun provenance - # so it gets properly logged in the environment provenance files. - env_mach_pes = case.get_env("mach_pes") - os.environ["OMP_NUM_THREADS"] = str(env_mach_pes.get_max_thread_count(case.get_values("COMP_CLASSES"))) - save_prerun_provenance(case) for cycle in range(data_assimilation_cycles): @@ -223,15 +263,22 @@ def case_run(case): case.set_value("CONTINUE_RUN", "TRUE") lid = new_lid() - pre_run_check(case, lid) - run_model(case) - post_run_check(case, lid) + if prerun_script: + do_external(prerun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"), + lid, prefix="prerun") + + lid = run_model(case, lid) save_logs(case, lid) # Copy log files back to caseroot if case.get_value("CHECK_TIMING") or case.get_value("SAVE_TIMING"): get_timing(case, lid) # Run the getTiming script if data_assimilation: - do_data_assimilation(data_assimilation_script, case.get_value("CASEROOT"), cycle, lid) + do_data_assimilation(data_assimilation_script, case.get_value("CASEROOT"), cycle, lid, + case.get_value("RUNDIR")) + + if postrun_script: + do_external(postrun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"), + lid, prefix="postrun") save_postrun_provenance(case) diff --git a/utils/python/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py similarity index 84% rename from utils/python/CIME/case_setup.py rename to scripts/lib/CIME/case_setup.py index 89af0208788..5ab6383373d 100644 --- a/utils/python/CIME/case_setup.py +++ b/scripts/lib/CIME/case_setup.py @@ -9,7 +9,7 @@ from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.machines import Machines from CIME.BuildTools.configure import configure -from CIME.utils import append_status, get_cime_root, get_model +from CIME.utils import get_cime_root, run_and_log_case_status, get_model from CIME.test_status import * import shutil @@ -59,11 +59,9 @@ def _build_usernl_files(case, model, comp): shutil.copy(model_nl, nlfile) ############################################################################### -def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): +def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, adjust_pio=True): ############################################################################### os.chdir(caseroot) - msg = "case.setup starting" - append_status(msg, caseroot=caseroot, sfile="CaseStatus") cimeroot = get_cime_root(case) @@ -71,7 +69,7 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): din_loc_root = case.get_value("DIN_LOC_ROOT") testcase = case.get_value("TESTCASE") expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"), - "inputdata root is not a directory: \"$din_loc_root\" ") + "inputdata root is not a directory: %s" % din_loc_root) # Check that userdefine settings are specified before expanding variable for vid, value in case: @@ -100,9 +98,6 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): logger.info("Successfully cleaned batch script case.run") - msg = "case.setup clean complete" - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - if not clean: case.load_env() @@ -138,6 +133,9 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): else: expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst, comp, ntasks)) + # Set TOTAL_CORES + case.set_value("TOTAL_CORES", case.total_tasks * case.cores_per_task ) + if os.path.exists("case.run"): logger.info("Machine/Decomp/Pes configuration has already been done ...skipping") else: @@ -152,10 +150,8 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): logger.debug("at update TOTALPES = %s"%pestot) case.set_value("TOTALPES", pestot) thread_count = env_mach_pes.get_max_thread_count(models) - if thread_count > 1: - case.set_value("BUILD_THREADED", True) - - expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"), + build_threaded = case.get_build_threaded() + expect(not (build_threaded and compiler == "nag"), "it is not possible to run with OpenMP if using the NAG Fortran compiler") cost_pes = env_mach_pes.get_cost_pes(pestot, thread_count, machine=case.get_value("MACH")) case.set_value("COST_PES", cost_pes) @@ -178,16 +174,8 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) # Make sure pio settings are consistant - for comp in models: - pio_stride = case.get_value("PIO_STRIDE_%s"%comp) - pio_numtasks = case.get_value("PIO_NUMTASKS_%s"%comp) - ntasks = case.get_value("NTASKS_%s"%comp) - if pio_stride < 0 or pio_stride > ntasks: - pio_stride = min(ntasks, tasks_per_node) - case.set_value("PIO_STRIDE_%s"%comp, pio_stride) - if pio_numtasks < 0 or pio_numtasks > ntasks: - pio_numtasks = max(1, ntasks//pio_stride) - case.set_value("PIO_NUMTASKS_%s"%comp, pio_numtasks) + if adjust_pio: + adjust_pio_layout(case, tasks_per_node) # Make a copy of env_mach_pes.xml in order to be able # to check that it does not change once case.setup is invoked @@ -199,6 +187,7 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): # Create user_nl files for the required number of instances if not os.path.exists("user_nl_cpl"): logger.info("Creating user_nl_xxx files for components and cpl") + # loop over models for model in models: comp = case.get_value("COMP_%s" % model) @@ -219,28 +208,48 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): logger.info("Generating component namelists as part of setup") create_namelists(case) - msg = "case.setup complete" - append_status(msg, caseroot=caseroot, sfile="CaseStatus") - # Record env information env_module = case.get_env("mach_specific") env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh") env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh") env_module.save_all_env_info("software_environment.txt") +def adjust_pio_layout(case, new_pio_stride): + + models = case.get_values("COMP_CLASSES") + for comp in models: + pio_stride = case.get_value("PIO_STRIDE_%s"%comp) + pio_numtasks = case.get_value("PIO_NUMTASKS_%s"%comp) + ntasks = case.get_value("NTASKS_%s"%comp) + new_stride = min(ntasks, new_pio_stride) + new_numtasks = max(1, ntasks//new_stride) + if pio_stride != new_stride: + logger.info("Resetting PIO_STRIDE_%s to %s"%(comp, new_stride)) + case.set_value("PIO_STRIDE_%s"%comp, new_stride) + if pio_numtasks != new_numtasks: + logger.info("Resetting PIO_NUMTASKS_%s to %s"%(comp, new_numtasks)) + case.set_value("PIO_NUMTASKS_%s"%comp, new_numtasks) + + ############################################################################### -def case_setup(case, clean=False, test_mode=False, reset=False, no_status=False): +def case_setup(case, clean=False, test_mode=False, reset=False, adjust_pio=True): ############################################################################### caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID") - if case.get_value("TEST") and not no_status: + phase = "setup.clean" if clean else "case.setup" + functor = lambda: _case_setup_impl(case, caseroot, clean, test_mode, reset, adjust_pio) + + if case.get_value("TEST") and not test_mode: test_name = casebaseid if casebaseid is not None else case.get_value("CASE") with TestStatus(test_dir=caseroot, test_name=test_name) as ts: try: - _case_setup_impl(case, caseroot, clean=clean, test_mode=test_mode, reset=reset) + run_and_log_case_status(functor, phase, caseroot=caseroot) except: ts.set_status(SETUP_PHASE, TEST_FAIL_STATUS) raise else: - ts.set_status(SETUP_PHASE, TEST_PASS_STATUS) + if clean: + ts.set_status(SETUP_PHASE, TEST_PEND_STATUS) + else: + ts.set_status(SETUP_PHASE, TEST_PASS_STATUS) else: - _case_setup_impl(case, caseroot, clean=clean, test_mode=test_mode, reset=reset) + run_and_log_case_status(functor, phase, caseroot=caseroot) diff --git a/utils/python/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py old mode 100755 new mode 100644 similarity index 94% rename from utils/python/CIME/case_st_archive.py rename to scripts/lib/CIME/case_st_archive.py index 24fa71d65a1..75158ee7212 --- a/utils/python/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -7,7 +7,7 @@ from CIME.XML.standard_module_setup import * from CIME.case_submit import submit from CIME.XML.env_archive import EnvArchive -from CIME.utils import append_status +from CIME.utils import run_and_log_case_status from os.path import isdir, join logger = logging.getLogger(__name__) @@ -198,11 +198,10 @@ def get_histfiles_for_restarts(case, archive, archive_entry, restfile): histfile = matchobj.group(1).strip() histfile = os.path.basename(histfile) # append histfile to the list ONLY if it exists in rundir before the archiving - if os.path.isfile(os.path.join(rundir,histfile)): + if os.path.isfile(os.path.join(rundir,histfile)): histfiles.append(histfile) return histfiles - ############################################################################### def _archive_restarts(case, archive, archive_entry, compclass, compname, datename, datename_is_last): @@ -353,9 +352,26 @@ def _archive_process(case, archive): _archive_history_files(case, archive, archive_entry, compclass, compname, histfiles_savein_rundir) +############################################################################### +def restore_from_archive(case): +############################################################################### + """ + Take most recent archived restart files and load them into current case. + """ + dout_sr = case.get_value("DOUT_S_ROOT") + rundir = case.get_value("RUNDIR") + most_recent_rest = run_cmd_no_fail("ls -1dt %s/rest/* | head -1" % dout_sr) + + for item in glob.glob("%s/*" % most_recent_rest): + base = os.path.basename(item) + dst = os.path.join(rundir, base) + if os.path.exists(dst): + os.remove(dst) + + shutil.copy(item, rundir) ############################################################################### -def case_st_archive(case): +def case_st_archive(case, no_resubmit=False): ############################################################################### """ Create archive object and perform short term archiving @@ -379,21 +395,15 @@ def case_st_archive(case): logger.info("st_archive starting") - # do short-term archiving - append_status("st_archiving starting", caseroot=caseroot, sfile="CaseStatus") - archive = EnvArchive(infile=os.path.join(caseroot, 'env_archive.xml')) + functor = lambda: _archive_process(case, archive) + run_and_log_case_status(functor, "st_archive", caseroot=caseroot) - _archive_process(case, archive) - - append_status("st_archiving completed", caseroot=caseroot, sfile="CaseStatus") logger.info("st_archive completed") # resubmit case if appropriate resubmit = case.get_value("RESUBMIT") - if resubmit > 0: - append_status("resubmitting from st_archive", - caseroot=caseroot, sfile="CaseStatus") + if resubmit > 0 and not no_resubmit: logger.info("resubmitting from st_archive, resubmit=%d"%resubmit) if case.get_value("MACH") == "mira": expect(os.path.isfile(".original_host"), "ERROR alcf host file not found") diff --git a/utils/python/CIME/case_submit.py b/scripts/lib/CIME/case_submit.py similarity index 74% rename from utils/python/CIME/case_submit.py rename to scripts/lib/CIME/case_submit.py index 684c4e886a1..d10032e56d7 100644 --- a/utils/python/CIME/case_submit.py +++ b/scripts/lib/CIME/case_submit.py @@ -7,11 +7,10 @@ """ import socket from CIME.XML.standard_module_setup import * -from CIME.utils import expect, append_status +from CIME.utils import expect, run_and_log_case_status from CIME.preview_namelists import create_namelists from CIME.check_lockedfiles import check_lockedfiles from CIME.check_input_data import check_all_input_data -from CIME.case_cmpgen_namelists import case_cmpgen_namelists from CIME.test_status import * logger = logging.getLogger(__name__) @@ -64,22 +63,31 @@ def _submit(case, job=None, resubmit=False, no_batch=False, batch_args=None): logger.warn("submit_jobs %s" % job) job_ids = case.submit_jobs(no_batch=no_batch, job=job, batch_args=batch_args) - msg = "Submitted jobs %s" % job_ids - append_status(msg, caseroot=caseroot, sfile="CaseStatus") + logger.info("Submitted job ids %s" % job_ids) def submit(case, job=None, resubmit=False, no_batch=False, batch_args=None): + if case.get_value("TEST"): + caseroot = case.get_value("CASEROOT") + casebaseid = case.get_value("CASEBASEID") + # This should take care of the race condition where the submitted job + # begins immediately and tries to set RUN phase. We proactively assume + # a passed SUBMIT phase. If this state is already PASS, don't set it again + # because then we'll lose RUN phase info if it's there. This info is important + # for system_tests_common to know if it needs to reinitialize the test or not. + with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts: + phase_status = ts.get_status(SUBMIT_PHASE) + if phase_status != TEST_PASS_STATUS: + ts.set_status(SUBMIT_PHASE, TEST_PASS_STATUS) + try: - _submit(case, job=job, resubmit=resubmit, no_batch=no_batch, batch_args=batch_args) + functor = lambda: _submit(case, job, resubmit, no_batch, batch_args) + run_and_log_case_status(functor, "case.submit", caseroot=case.get_value("CASEROOT")) except: # If something failed in the batch system, make sure to mark # the test as failed if we are running a test. if case.get_value("TEST"): - caseroot = case.get_value("CASEROOT") - casebaseid = case.get_value("CASEBASEID") - with TestStatus(test_dir=caseroot, test_name=casebaseid, lock=True) as ts: - ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="batch system failure") - - append_status("Batch submission failed, TestStatus file changed to read-only", caseroot=caseroot, sfile="TestStatus.log") + with TestStatus(test_dir=caseroot, test_name=casebaseid) as ts: + ts.set_status(SUBMIT_PHASE, TEST_FAIL_STATUS) raise @@ -88,9 +96,6 @@ def check_case(case, caseroot): create_namelists(case) # Must be called before check_all_input_data logger.info("Checking that inputdata is available as part of case submission") check_all_input_data(case) - # Now that we have baselines, do baseline operations - if case.get_value("TEST"): - case_cmpgen_namelists(case) expect(case.get_value("BUILD_COMPLETE"), "Build complete is " "not True please rebuild the model by calling case.build") diff --git a/utils/python/CIME/case_test.py b/scripts/lib/CIME/case_test.py similarity index 93% rename from utils/python/CIME/case_test.py rename to scripts/lib/CIME/case_test.py index 2bfafe34c08..85aaed7149a 100644 --- a/utils/python/CIME/case_test.py +++ b/scripts/lib/CIME/case_test.py @@ -3,7 +3,7 @@ """ from CIME.XML.standard_module_setup import * -from CIME.utils import expect, find_system_test, append_status, find_proc_id +from CIME.utils import expect, find_system_test, append_testlog, find_proc_id from CIME.SystemTests.system_tests_common import * import sys, signal @@ -61,8 +61,8 @@ def case_test(case, testname=None): caseroot = case.get_value("CASEROOT") with TestStatus(test_dir=caseroot) as ts: ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize") - append_status(str(sys.exc_info()[1]), sfile="TestStatus.log") - raise + append_testlog(str(sys.exc_info()[1])) + return False success = test.run() diff --git a/utils/python/CIME/check_input_data.py b/scripts/lib/CIME/check_input_data.py similarity index 100% rename from utils/python/CIME/check_input_data.py rename to scripts/lib/CIME/check_input_data.py diff --git a/utils/python/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py similarity index 96% rename from utils/python/CIME/check_lockedfiles.py rename to scripts/lib/CIME/check_lockedfiles.py index 2a6a7f22ac0..ee99a37398b 100644 --- a/utils/python/CIME/check_lockedfiles.py +++ b/scripts/lib/CIME/check_lockedfiles.py @@ -9,6 +9,8 @@ from CIME.XML.env_batch import EnvBatch from CIME.utils import run_cmd_no_fail +logger = logging.getLogger(__name__) + import glob, shutil LOCKED_DIR = "LockedFiles" @@ -82,13 +84,15 @@ def check_lockedfiles(caseroot=None): If caseroot is not specified, it is set to the current working directory """ caseroot = os.getcwd() if caseroot is None else caseroot - lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "[^.]*.xml")) + lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml")) for lfile in lockedfiles: fpart = os.path.basename(lfile) + # ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name + if lfile.count('.') > 1: + continue cfile = os.path.join(caseroot, fpart) if os.path.isfile(cfile): objname = fpart.split('.')[0] - logging.info("Checking file %s"%objname) if objname == "env_build": f1obj = EnvBuild(caseroot, cfile) f2obj = EnvBuild(caseroot, lfile) @@ -104,7 +108,6 @@ def check_lockedfiles(caseroot=None): else: logging.warn("Locked XML file '%s' is not current being handled" % fpart) continue - diffs = f1obj.compare_xml(f2obj) if diffs: logging.warn("File %s has been modified"%lfile) diff --git a/utils/python/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py similarity index 96% rename from utils/python/CIME/code_checker.py rename to scripts/lib/CIME/code_checker.py index 66b0055c48b..2984df3c2fd 100644 --- a/utils/python/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -24,7 +24,7 @@ def _run_pylint(on_file, interactive): # add init-hook option cmd_options += " --init-hook='sys.path.extend((\"%s\",\"%s\",\"%s\"))'"%\ - (os.path.join(cimeroot,"utils","python"), + (os.path.join(cimeroot,"scripts","lib"), os.path.join(cimeroot,"scripts","Tools"), os.path.join(cimeroot,"tools","unit_testing","python")) @@ -65,7 +65,8 @@ def _should_pylint_skip(filepath): def get_all_checkable_files(): ############################################################################### cimeroot = get_cime_root() - all_git_files = run_cmd_no_fail("git ls-files --full-name %s" % cimeroot, verbose=False).splitlines() + all_git_files = run_cmd_no_fail("git ls-files", from_dir=cimeroot, verbose=False).splitlines() + files_to_test = [item for item in all_git_files if ((item.endswith(".py") or is_python_executable(os.path.join(cimeroot, item))) and not _should_pylint_skip(item))] return files_to_test diff --git a/utils/python/CIME/compare_namelists.py b/scripts/lib/CIME/compare_namelists.py similarity index 100% rename from utils/python/CIME/compare_namelists.py rename to scripts/lib/CIME/compare_namelists.py diff --git a/utils/python/CIME/compare_test_results.py b/scripts/lib/CIME/compare_test_results.py similarity index 90% rename from utils/python/CIME/compare_test_results.py rename to scripts/lib/CIME/compare_test_results.py index 6720063c51f..4869be8919d 100644 --- a/utils/python/CIME/compare_test_results.py +++ b/scripts/lib/CIME/compare_test_results.py @@ -1,5 +1,5 @@ import CIME.compare_namelists, CIME.simple_compare -from CIME.utils import expect, get_model +from CIME.utils import expect, get_model, append_status from CIME.test_status import * from CIME.hist_utils import compare_baseline from CIME.case_cmpgen_namelists import case_cmpgen_namelists @@ -67,12 +67,12 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test ts = TestStatus(test_dir=test_dir) test_name = ts.get_name() if (compare_tests in [[], None] or CIME.utils.match_any(test_name, compare_tests)): - CIME.utils.append_status( - msg = ("Comparing against baseline with compare_test_results:\n" + - "Baseline: %s\n"%(baseline_name) + - "In baseline_root: %s"%(baseline_root)), - caseroot = test_dir, - sfile = logfile_name) + append_status( + "Comparing against baseline with compare_test_results:\n" + + "Baseline: %s\n"%(baseline_name) + + "In baseline_root: %s"%(baseline_root), + logfile_name, + caseroot=test_dir) if (not hist_only): nl_compare_result = None @@ -143,15 +143,9 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test print brief_result, - CIME.utils.append_status( - msg = brief_result, - caseroot = test_dir, - sfile = logfile_name) + append_status(brief_result, logfile_name, caseroot=test_dir) if detailed_comments: - CIME.utils.append_status( - msg = "Detailed comments:\n" + detailed_comments, - caseroot = test_dir, - sfile = logfile_name) + append_status("Detailed comments:\n" + detailed_comments, logfile_name, caseroot=test_dir) return all_pass_or_skip diff --git a/utils/python/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py similarity index 98% rename from utils/python/CIME/get_timing.py rename to scripts/lib/CIME/get_timing.py index b914b63c790..63b44bb2cf0 100644 --- a/utils/python/CIME/get_timing.py +++ b/scripts/lib/CIME/get_timing.py @@ -122,9 +122,9 @@ def getTiming(self): ncpl_base_period = self.case.get_value("NCPL_BASE_PERIOD") atm_ncpl = self.case.get_value("ATM_NCPL") ocn_ncpl = self.case.get_value("OCN_NCPL") - ccsm_compset = self.case.get_value("COMPSET") - if ccsm_compset is None: - ccsm_compset = "" + compset = self.case.get_value("COMPSET") + if compset is None: + compset = "" grid = self.case.get_value("GRID") run_type = self.case.get_value("RUN_TYPE") stop_option = self.case.get_value("STOP_OPTION") @@ -232,7 +232,7 @@ def getTiming(self): self.write(" Curr Date : %s\n" % now) self.write(" grid : %s\n" % grid) - self.write(" compset : %s\n" % ccsm_compset) + self.write(" compset : %s\n" % compset) self.write(" run_type : %s, continue_run = %s " "(inittype = %s)\n" % (run_type, str(continue_run).upper(), inittype)) @@ -247,7 +247,7 @@ def getTiming(self): self.write(" --------- ------ ------- ------ " "------ --------- ------ \n") maxthrds = 0 - for k in ['CPL', 'GLC', 'WAV', 'LND', 'ROF', 'ICE', 'ATM', 'OCN']: + for k in self.case.get_values("COMP_CLASSES"): m = self.models[k] self.write(" %s = %-8s %-6u %-6u %-6u x %-6u " "%-6u (%-6u) \n" @@ -335,7 +335,7 @@ def getTiming(self): self.write(" TOT Run Time: %10.3f seconds %10.3f seconds/mday " "%10.2f myears/wday \n" % (tmax, tmax/adays, tmaxr)) - for k in ['LND', 'ROF', 'ICE', 'ATM', 'OCN', 'GLC', 'WAV', 'CPL']: + for k in self.case.get_values("COMP_CLASSES"): m = self.models[k] self.write(" %s Run Time: %10.3f seconds " "%10.3f seconds/mday %10.2f myears/wday \n" diff --git a/utils/python/CIME/hist_utils.py b/scripts/lib/CIME/hist_utils.py similarity index 98% rename from utils/python/CIME/hist_utils.py rename to scripts/lib/CIME/hist_utils.py index 3d43ff4118d..5ae1217f57c 100644 --- a/utils/python/CIME/hist_utils.py +++ b/scripts/lib/CIME/hist_utils.py @@ -263,10 +263,14 @@ def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False, outfil if mstr1 != mstr2: mstr = mstr1+mstr2 - output_filename = "%s%s.cprnc.out"%(basename, mstr) + output_filename = os.path.join(rundir, "%s%s.cprnc.out" % (basename, mstr)) if outfile_suffix: - output_filename += ".%s"%(outfile_suffix) - cpr_stat, out, _ = run_cmd("%s -m %s %s 2>&1 | tee %s/%s" % (cprnc_exe, file1, file2, rundir, output_filename)) + output_filename += ".%s" % outfile_suffix + + cpr_stat = run_cmd("%s -m %s %s" % (cprnc_exe, file1, file2), combine_output=True, arg_stdout=output_filename)[0] + with open(output_filename, "r") as fd: + out = fd.read() + if multiinst_cpl_compare: # In a multiinstance test the cpl hist file will have a different number of # dimensions and so cprnc will indicate that the files seem to be DIFFERENT diff --git a/utils/python/CIME/namelist.py b/scripts/lib/CIME/namelist.py similarity index 88% rename from utils/python/CIME/namelist.py rename to scripts/lib/CIME/namelist.py index f2f02cf752a..d91a31184c3 100644 --- a/utils/python/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -101,6 +101,7 @@ # pylint: disable=line-too-long,too-many-lines,invalid-name import re +import collections # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import @@ -112,7 +113,15 @@ # Fortran syntax regular expressions. # Variable names. -FORTRAN_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{0,62}$", re.IGNORECASE) +#FORTRAN_NAME_REGEX = re.compile(r"(^[a-z][a-z0-9_]{0,62})(\([+-]?\d*:?[+-]?\d*:?[+-]?\d*\))?$", re.IGNORECASE) +FORTRAN_NAME_REGEX = re.compile(r"""(^[a-z][a-z0-9_]{0,62}) # The variable name + (\( # begin optional index expression + (([+-]?\d+) # Single valued index + | # or + (([+-]?\d+)?:([+-]?\d+)?:?([+-]?\d+)?)) # colon seperated triplet + \))?\s*$""" # end optional index expression + , re.IGNORECASE | re.VERBOSE) + FORTRAN_LITERAL_REGEXES = {} # Integer literals. _int_re_string = r"(\+|-)?[0-9]+" @@ -149,6 +158,7 @@ def is_valid_fortran_name(string): 1. The name must start with a letter. 2. All characters in a name must be alphanumeric (or underscores). 3. The maximum name length is 63 characters. + 4. We only handle a single dimension !!! >>> is_valid_fortran_name("") False @@ -156,6 +166,20 @@ def is_valid_fortran_name(string): True >>> is_valid_fortran_name("A") True + >>> is_valid_fortran_name("A(4)") + True + >>> is_valid_fortran_name("A(::)") + True + >>> is_valid_fortran_name("A(1:2:3)") + True + >>> is_valid_fortran_name("A(1::)") + True + >>> is_valid_fortran_name("A(:-2:)") + True + >>> is_valid_fortran_name("A(1::+3)") + True + >>> is_valid_fortran_name("A(1,3)") + False >>> is_valid_fortran_name("2") False >>> is_valid_fortran_name("_") @@ -171,6 +195,62 @@ def is_valid_fortran_name(string): """ return FORTRAN_NAME_REGEX.search(string) is not None +def get_fortran_name_only(full_var): + """ remove array section if any and return only the variable name + >>> get_fortran_name_only('foo') + 'foo' + >>> get_fortran_name_only('foo(3)') + 'foo' + >>> get_fortran_name_only('foo(::)') + 'foo' + >>> get_fortran_name_only('foo(1::)') + 'foo' + >>> get_fortran_name_only('foo(:+2:)') + 'foo' + >>> get_fortran_name_only('foo(::-3)') + 'foo' + >>> get_fortran_name_only('foo(::)') + 'foo' + """ + m = FORTRAN_NAME_REGEX.search(full_var) + return m.group(1) + +def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): + """ get indices from a fortran namelist variable as a triplet of minindex, maxindex and step + + >>> get_fortran_variable_indices('foo(3)') + (3, 3, 1) + >>> get_fortran_variable_indices('foo(1:2:3)') + (1, 2, 3) + >>> get_fortran_variable_indices('foo(::)', varlen=4) + (1, 4, 1) + >>> get_fortran_variable_indices('foo(::2)', varlen=4) + (1, 4, 2) + >>> get_fortran_variable_indices('foo(::)', allow_any_len=True) + (1, -1, 1) + """ + m = FORTRAN_NAME_REGEX.search(varname) + (minindex, maxindex, step) = (1, varlen, 1) + + if m.group(4) is not None: + minindex = int(m.group(4)) + maxindex = minindex + step = 1 + + elif m.group(5) is not None: + if m.group(6) is not None: + minindex = int(m.group(6)) + if m.group(7) is not None: + maxindex = int(m.group(7)) + if m.group(8) is not None: + step = int(m.group(8)) + + if allow_any_len and maxindex == minindex: + maxindex = -1 + + expect(step != 0,"Step size 0 not allowed") + + return (minindex, maxindex, step) def fortran_namelist_base_value(string): r"""Strip off whitespace and repetition syntax from a namelist value. @@ -620,6 +700,7 @@ def expand_literal_list(literals): expanded += int(num) * [value] else: expanded.append(literal) + return expanded @@ -701,6 +782,7 @@ def merge_literal_lists(default, overwrite): merged = [] default = expand_literal_list(default) overwrite = expand_literal_list(overwrite) + for default_elem, elem in zip(default, overwrite): if elem == '': merged.append(default_elem) @@ -804,12 +886,15 @@ def __init__(self, groups=None): for group_name in groups: expect(group_name is not None, " Got None in groups %s"%groups) group_lc = group_name.lower() - self._groups[group_lc] = {} + self._groups[group_lc] = collections.OrderedDict() for variable_name in groups[group_name]: variable_lc = variable_name.lower() self._groups[group_lc][variable_lc] = \ groups[group_name][variable_name] + def clean_groups(self): + self._groups = collections.OrderedDict() + def get_group_names(self): """Return a list of all groups in the namelist. @@ -827,9 +912,15 @@ def get_variable_names(self, group_name): >>> Namelist().get_variable_names('foo') [] + >>> x = parse(text='&foo bar=,bazz=true,bazz(2)=fred,bang=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bang', u'bar', u'bazz', u'bazz(2)'] >>> x = parse(text='&foo bar=,bazz=true,bang=6*""/') >>> sorted(x.get_variable_names('fOo')) [u'bang', u'bar', u'bazz'] + >>> x = parse(text='&foo bar(::)=,bazz=false,bazz(2)=true,bazz(:2:)=6*""/') + >>> sorted(x.get_variable_names('fOo')) + [u'bar(::)', u'bazz', u'bazz(2)', u'bazz(:2:)'] """ group_name = group_name.lower() if group_name not in self._groups: @@ -855,6 +946,7 @@ def get_variable_value(self, group_name, variable_name): if group_name not in self._groups or \ variable_name not in self._groups[group_name]: return [u''] + return self._groups[group_name][variable_name] def get_value(self, variable_name): @@ -870,6 +962,8 @@ def get_value(self, variable_name): SystemExit: ERROR: Namelist.get_value: Variable %s is present in multiple groups: [u'bazz', u'foo'] >>> parse(text='&foo bar=1 / &bazz /').get_value('Bar') [u'1'] + >>> parse(text='&foo bar(2)=1 / &bazz /').get_value('Bar(2)') + [u'1'] >>> parse(text='&foo / &bazz /').get_value('bar') [u''] """ @@ -884,25 +978,54 @@ def get_value(self, variable_name): else: return [u''] - def set_variable_value(self, group_name, variable_name, value): + def set_variable_value(self, group_name, variable_name, value, var_size=1): """Set the value of the specified variable. >>> x = parse(text='&foo bar=1 /') + >>> x.get_variable_value('foo', 'bar') + [u'1'] + >>> x.set_variable_value('foo', 'bar(2)', [u'3'], var_size=4) + >>> x.get_variable_value('foo', 'bar') + [u'1', u'3'] >>> x.set_variable_value('foo', 'bar', [u'2']) + >>> x.get_variable_value('foo', 'bar') + [u'2', u'3'] >>> x.set_variable_value('foo', 'bazz', [u'3']) >>> x.set_variable_value('Brack', 'baR', [u'4']) - >>> x.get_variable_value('foo', 'bar') - [u'2'] >>> x.get_variable_value('foo', 'bazz') [u'3'] >>> x.get_variable_value('brack', 'bar') [u'4'] + >>> x.set_variable_value('foo', 'red(2:6:2)', [u'2', u'4', u'6'], var_size=12) + >>> x.get_variable_value('foo', 'red') + ['', u'2', '', u'4', '', u'6'] """ group_name = group_name.lower() - variable_name = variable_name.lower() + + minindex, maxindex, step = get_fortran_variable_indices(variable_name, var_size) + variable_name = get_fortran_name_only(variable_name.lower()) + + expect(minindex > 0, "Indices < 1 not supported in CIME interface to fortran namelists... lower bound=%s"%minindex) + if group_name not in self._groups: self._groups[group_name] = {} - self._groups[group_name][variable_name] = value + tlen = 1 + if variable_name in self._groups[group_name]: + tlen = len(self._groups[group_name][variable_name]) + else: + tlen = 1 + self._groups[group_name][variable_name] = [''] + + if minindex > tlen: + self._groups[group_name][variable_name].extend(['']*(minindex-tlen-1)) + + for i in range(minindex, maxindex+2*step, step): + while len(self._groups[group_name][variable_name]) < i: + self._groups[group_name][variable_name].append('') + self._groups[group_name][variable_name][i-1] = value.pop(0) + if len(value) == 0: + break + def delete_variable(self, group_name, variable_name): """Delete a variable from a specified group. @@ -933,6 +1056,8 @@ def merge_nl(self, other, overwrite=False): >>> x = parse(text='&foo bar=1 bazz=,2 brat=3/') >>> y = parse(text='&foo bar=2 bazz=3*1 baker=4 / &foo2 barter=5 /') + >>> y.get_value('bazz') + [u'1', u'1', u'1'] >>> x.merge_nl(y) >>> sorted(x.get_group_names()) [u'foo', u'foo2'] @@ -950,7 +1075,6 @@ def merge_nl(self, other, overwrite=False): [u'4'] >>> x.get_value('barter') [u'5'] - >>> x = parse(text='&foo bar=1 bazz=,2 brat=3/') >>> y = parse(text='&foo bar=2 bazz=3*1 baker=4 / &foo2 barter=5 /') >>> x.merge_nl(y, overwrite=True) @@ -981,7 +1105,8 @@ def merge_nl(self, other, overwrite=False): merged_val = merge_literal_lists(self_val, other_val) else: merged_val = merge_literal_lists(other_val, self_val) - self.set_variable_value(group_name, variable_name, merged_val) + self.set_variable_value(group_name, variable_name, merged_val, + var_size=len(merged_val)) def write(self, out_file, groups=None, append=False, format_='nml', sorted_groups=True): """Write a Fortran namelist to a file. @@ -1029,7 +1154,7 @@ def _write(self, out_file, groups, format_, sorted_groups): values = group[name] # @ is used in a namelist to put the same namelist variable in multiple groups - # in the write phase, all characters in the namelist variable name after + # in the write phase, all characters in the namelist variable name after # the @ and including the @ should be removed if "@" in name: name = re.sub('@.+$', "", name) @@ -1120,7 +1245,10 @@ def __init__(self, text, groupless=False): # Dictionary with group names as keys, and dictionaries of variable # name-value pairs as values. (Or a single flat dictionary if # `groupless=True`.) - self._settings = {} + self._settings = collections.OrderedDict() + # Fortran allows setting a particular index of an array + # such as foo(2)='k' + # this dict is set to that value if used. self._groupless = groupless def _line_col_string(self): @@ -1143,7 +1271,7 @@ def _next(self): >>> _NamelistParser(' ')._next() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 1) + _NamelistEOF: Unexpected end of file encountered in namelist. """ # If at the end of the file, we should raise _NamelistEOF. The easiest # way to do this is to just advance. @@ -1185,11 +1313,11 @@ def _advance(self, nchars=1, check_eof=False): >>> x._advance(1) Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 3, column 2) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('abc\n')._advance(4) Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 2, column 0) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser('ab') >>> x._advance(check_eof=True) False @@ -1214,7 +1342,7 @@ def _advance(self, nchars=1, check_eof=False): if check_eof: return end_of_file elif end_of_file: - raise _NamelistEOF(message="At "+self._line_col_string()) + raise _NamelistEOF(message=None) def _eat_whitespace(self, allow_initial_comment=False): r"""Advance until the next non-whitespace character. @@ -1235,7 +1363,7 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 2, column 3) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser(' \n! blah\n ! blah\n a') >>> x._eat_whitespace() True @@ -1292,12 +1420,12 @@ def _eat_comment(self): >>> x._eat_comment() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 3, column 8) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser('! foo\n') >>> x._eat_comment() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 2, column 0) + _NamelistEOF: Unexpected end of file encountered in namelist. """ if self._curr() != '!': return False @@ -1324,7 +1452,7 @@ def _expect_char(self, chars): >>> x._expect_char('a') Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected 'a' but found 'b' at line 1, column 1 + _NamelistParseError: Error in parsing namelist: expected 'a' but found 'b' >>> x._expect_char('ab') """ if self._curr() not in chars: @@ -1332,9 +1460,8 @@ def _expect_char(self, chars): char_description = repr(str(chars)) else: char_description = "one of the characters in %r" % str(chars) - raise _NamelistParseError("expected %s but found %r at %s" % - (char_description, str(self._curr()), - self._line_col_string())) + raise _NamelistParseError("expected %s but found %r" % + (char_description, str(self._curr()))) def _parse_namelist_group_name(self): r"""Parses and returns a namelist group name at the current position. @@ -1342,11 +1469,11 @@ def _parse_namelist_group_name(self): >>> _NamelistParser('abc')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected '&' but found 'a' at line 1, column 0 + _NamelistParseError: Error in parsing namelist: expected '&' but found 'a' >>> _NamelistParser('&abc')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 4) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('&abc ')._parse_namelist_group_name() u'abc' >>> _NamelistParser('&abc\n')._parse_namelist_group_name() @@ -1354,15 +1481,15 @@ def _parse_namelist_group_name(self): >>> _NamelistParser('&abc/ ')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: 'abc/' is not a valid variable name at line 1, column 5 + _NamelistParseError: Error in parsing namelist: 'abc/' is not a valid variable name >>> _NamelistParser('&abc= ')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: 'abc=' is not a valid variable name at line 1, column 5 + _NamelistParseError: Error in parsing namelist: 'abc=' is not a valid variable name >>> _NamelistParser('& ')._parse_namelist_group_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: '' is not a valid variable name at line 1, column 1 + _NamelistParseError: Error in parsing namelist: '' is not a valid variable name """ self._expect_char("&") self._advance() @@ -1378,32 +1505,48 @@ def _parse_variable_name(self, allow_equals=True): >>> _NamelistParser('abc')._parse_variable_name() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 3) + _NamelistEOF: Unexpected end of file encountered in namelist. + >>> _NamelistParser('foo(2)= ')._parse_variable_name() + u'foo(2)' >>> _NamelistParser('abc ')._parse_variable_name() u'abc' >>> _NamelistParser('ABC ')._parse_variable_name() u'abc' >>> _NamelistParser('abc\n')._parse_variable_name() u'abc' + >>> _NamelistParser('abc%fred\n')._parse_variable_name() + u'abc%fred' + >>> _NamelistParser('abc(2)@fred\n')._parse_variable_name() + u'abc(2)@fred' + >>> _NamelistParser('abc(1:2:3)\n')._parse_variable_name() + u'abc(1:2:3)' >>> _NamelistParser('abc=')._parse_variable_name() u'abc' + >>> _NamelistParser('abc(1,2) ')._parse_variable_name() + Traceback (most recent call last): + ... + _NamelistParseError: Error in parsing namelist: Multiple dimensions not supported in CIME namelist variables 'abc(1,2)' >>> _NamelistParser('abc, ')._parse_variable_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: 'abc,' is not a valid variable name at line 1, column 4 + _NamelistParseError: Error in parsing namelist: 'abc,' is not a valid variable name >>> _NamelistParser(' ')._parse_variable_name() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: '' is not a valid variable name at line 1, column 0 + _NamelistParseError: Error in parsing namelist: '' is not a valid variable name """ old_pos = self._pos separators = (' ', '\n', '=') if allow_equals else (' ', '\n') while self._curr() not in separators: self._advance() text = self._text[old_pos:self._pos] + if '(' in text: + expect(')' in text,"Parsing error ") + elif ')' in text: + expect(False,"Parsing error ") # @ is used in a namelist to put the same namelist variable in multiple groups - # in the write phase, all characters in the namelist variable name after + # in the write phase, all characters in the namelist variable name after # the @ and including the @ should be removed if "%" in text: text_check = re.sub('%.+$', "", text) @@ -1411,10 +1554,16 @@ def _parse_variable_name(self, allow_equals=True): text_check = re.sub('@.+$', "", text) else: text_check = text + if not is_valid_fortran_name(text_check): - raise _NamelistParseError("%r is not a valid variable name at %s" % - (str(text), self._line_col_string())) - return text.lower() + if re.search(r".*\(.*\,.*\)", text_check): + err_str = "Multiple dimensions not supported in CIME namelist variables %r" % str(text) + else: + err_str = "%r is not a valid variable name" % str(text) + raise _NamelistParseError(err_str) + name = text.lower() + + return name def _parse_character_literal(self): """Parse and return a character literal (a string). @@ -1425,7 +1574,7 @@ def _parse_character_literal(self): >>> _NamelistParser('"abc')._parse_character_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 4) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc" ')._parse_character_literal() u'"abc"' >>> _NamelistParser("'abc' ")._parse_character_literal() @@ -1433,7 +1582,7 @@ def _parse_character_literal(self): >>> _NamelistParser("*abc* ")._parse_character_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal at line 1, column 4 + _NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal >>> _NamelistParser("'abc''def' ")._parse_character_literal() u"'abc''def'" >>> _NamelistParser("'abc''' ")._parse_character_literal() @@ -1457,8 +1606,8 @@ def _parse_character_literal(self): break text = self._text[old_pos:self._pos+1] if not is_valid_fortran_namelist_literal("character", text): - raise _NamelistParseError("%s is not a valid character literal at %s" % - (text, self._line_col_string())) + raise _NamelistParseError("%s is not a valid character literal" % + (text)) return text def _parse_complex_literal(self): @@ -1470,21 +1619,21 @@ def _parse_complex_literal(self): >>> _NamelistParser('(1.,2.')._parse_complex_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 6) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.) ')._parse_complex_literal() u'(1.,2.)' >>> _NamelistParser("(A,B) ")._parse_complex_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: '(A,B)' is not a valid complex literal at line 1, column 4 + _NamelistParseError: Error in parsing namelist: '(A,B)' is not a valid complex literal """ old_pos = self._pos while self._curr() != ')': self._advance() text = self._text[old_pos:self._pos+1] if not is_valid_fortran_namelist_literal("complex", text): - raise _NamelistParseError("%r is not a valid complex literal at %s" - % (str(text), self._line_col_string())) + raise _NamelistParseError("%r is not a valid complex literal" + % (str(text))) return text def _look_ahead_for_equals(self, pos): @@ -1536,7 +1685,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('"abc"')._parse_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 5) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc"')._parse_literal(allow_eof_end=True) u'"abc"' >>> _NamelistParser('(1.,2.) ')._parse_literal() @@ -1544,7 +1693,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('(1.,2.)')._parse_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 7) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.)')._parse_literal(allow_eof_end=True) u'(1.,2.)' >>> _NamelistParser('5 ')._parse_literal() @@ -1560,7 +1709,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('hamburger ')._parse_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' at line 1, column 9 + _NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' >>> _NamelistParser('5,')._parse_literal() u'5' >>> _NamelistParser('5\n')._parse_literal() @@ -1578,13 +1727,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('6*')._parse_literal() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 2) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('6*')._parse_literal(allow_eof_end=True) u'6*' >>> _NamelistParser('foo= ')._parse_literal() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' at line 1, column 4 + _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' >>> _NamelistParser('5,')._parse_literal(allow_name=True) u'5' >>> x = _NamelistParser('foo= ') @@ -1594,7 +1743,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> _NamelistParser('6*foo= ')._parse_literal(allow_name=True) Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' at line 1, column 6 + _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) >>> x._curr() @@ -1634,7 +1783,13 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): if allow_name: separators.append('=') while new_pos != self._len and self._text[new_pos] not in separators: + # allow commas if they are inside () + if self._text[new_pos] == '(': + separators.remove(',') + elif self._text[new_pos] == ')': + separators.append(',') new_pos += 1 + if not allow_eof_end and new_pos == self._len: # At the end of the file, give up by throwing an EOF. self._advance(self._len) @@ -1646,8 +1801,8 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): text = self._text[old_pos:self._pos] if not any(is_valid_fortran_namelist_literal(type_, text) for type_ in ("integer", "logical", "real")): - raise _NamelistParseError("expected literal value, but got %r at %s" - % (str(text), self._line_col_string())) + raise _NamelistParseError("expected literal value, but got %r" + % (str(text))) return text def _expect_separator(self, allow_eof=False): @@ -1690,7 +1845,7 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected one of the characters in ' \n,/' but found 'a' at line 1, column 0 + _NamelistParseError: Error in parsing namelist: expected one of the characters in ' \n,/' but found 'a' >>> x = _NamelistParser(" , a") >>> x._expect_separator() True @@ -1723,9 +1878,9 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator(allow_eof=True) Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: found group-terminating '/' in file without group names at line 1, column 1 + _NamelistParseError: Error in parsing namelist: found group-terminating '/' in file without group names """ - errstring = "found group-terminating '/' in file without group names at " + errstring = "found group-terminating '/' in file without group names" # Deal with the possibility that we are already at EOF. if allow_eof and self._pos == self._len: return False @@ -1735,8 +1890,7 @@ def _expect_separator(self, allow_eof=False): self._eat_whitespace() if self._curr() == '/': if allow_eof: - raise _NamelistParseError(errstring + - self._line_col_string()) + raise _NamelistParseError(errstring) else: return False except _NamelistEOF: @@ -1766,6 +1920,8 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo='bar' /")._parse_name_and_values() (u'foo', [u"'bar'"]) + >>> _NamelistParser("foo(3)='bar' /")._parse_name_and_values() + (u'foo(3)', [u"'bar'"]) >>> _NamelistParser("foo ='bar' /")._parse_name_and_values() (u'foo', [u"'bar'"]) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() @@ -1773,23 +1929,27 @@ def _parse_name_and_values(self, allow_eof_end=False): >>> _NamelistParser("foo 'bar' /")._parse_name_and_values() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected '=' but found "'" at line 1, column 4 + _NamelistParseError: Error in parsing namelist: expected '=' but found "'" >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() (u'foo', [u"'bar'", u"'bazz'"]) >>> _NamelistParser("foo=,,'bazz',6*/")._parse_name_and_values() (u'foo', [u'', u'', u"'bazz'", u'6*']) >>> _NamelistParser("foo='bar' 'bazz' foo2='ban'")._parse_name_and_values() (u'foo', [u"'bar'", u"'bazz'"]) + >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() + (u'foo', [u"'bar'", u"'bazz'"]) >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() Traceback (most recent call last): ... - _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" at line 1, column 15 + _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'', u'', u"'bazz'", u'6*']) + >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) + (u'foo(3)', [u"'bazz'"]) >>> _NamelistParser("foo=")._parse_name_and_values() Traceback (most recent call last): ... - _NamelistEOF: Unexpected end of file encountered in namelist. (At line 1, column 4) + _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'']) >>> _NamelistParser("foo= ")._parse_name_and_values(allow_eof_end=True) @@ -1798,10 +1958,15 @@ def _parse_name_and_values(self, allow_eof_end=False): (u'foo', [u'2']) >>> _NamelistParser("foo=1,2")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'1', u'2']) + >>> _NamelistParser("foo(1:2)=1,2,3 ")._parse_name_and_values(allow_eof_end=True) + Traceback (most recent call last): + ... + SystemExit: ERROR: Too many values for array foo(1:2) >>> _NamelistParser("foo=1,")._parse_name_and_values(allow_eof_end=True) (u'foo', [u'1', u'']) """ name = self._parse_variable_name() + self._eat_whitespace() self._expect_char("=") try: @@ -1824,6 +1989,11 @@ def _parse_name_and_values(self, allow_eof_end=False): break # and if it really is a literal, add it. values.append(literal) + (minindex, maxindex, step) = get_fortran_variable_indices(name,allow_any_len=True) + if (minindex > 1 or maxindex > minindex or step > 1) and maxindex > 0: + arraylen =max(0,1 + ((maxindex - minindex)/step)) + expect(len(values) <= arraylen, "Too many values for array %s"%(name)) + return name, values def _parse_namelist_group(self): @@ -1835,17 +2005,17 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {}} + OrderedDict([(u'group', {})]) >>> x._curr() u'/' >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']}} + OrderedDict([(u'group', {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']})]) >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']} + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'5', u'5'])]) >>> x._curr() u'/' >>> x = _NamelistParser("&group /&group /") @@ -1858,15 +2028,15 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group foo='bar', foo='bazz' /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bazz'"]}} + OrderedDict([(u'group', {u'foo': [u"'bazz'"]})]) >>> x = _NamelistParser("&group foo='bar', foo= /") >>> x._parse_namelist_group() >>> x._settings - {u'group': {u'foo': [u"'bar'"]}} + OrderedDict([(u'group', {u'foo': [u"'bar'"]})]) >>> x = _NamelistParser("&group foo='bar', foo= /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - {u'foo': [u"'bar'"]} + OrderedDict([(u'foo', [u"'bar'"])]) """ group_name = self._parse_namelist_group_name() if not self._groupless: @@ -1878,14 +2048,17 @@ def _parse_namelist_group(self): self._eat_whitespace() while self._curr() != '/': name, values = self._parse_name_and_values() + dsettings = [] if self._groupless: if name in self._settings: - values = merge_literal_lists(self._settings[name], values) + dsettings = self._settings[name] + values = merge_literal_lists(dsettings, values) self._settings[name] = values else: group = self._settings[group_name] if name in group: - values = merge_literal_lists(group[name], values) + dsettings = group[name] + values = merge_literal_lists(dsettings, values) group[name] = values def parse_namelist(self): @@ -1895,27 +2068,33 @@ def parse_namelist(self): first by namelist group name, then by variable name. >>> _NamelistParser("").parse_namelist() - {} + OrderedDict() >>> _NamelistParser(" \n!Comment").parse_namelist() - {} + OrderedDict() >>> _NamelistParser(" &group /").parse_namelist() - {u'group': {}} + OrderedDict([(u'group', {})]) >>> _NamelistParser("! Comment \n &group /! Comment\n ").parse_namelist() - {u'group': {}} + OrderedDict([(u'group', {})]) >>> _NamelistParser("! Comment \n &group /! Comment ").parse_namelist() - {u'group': {}} + OrderedDict([(u'group', {})]) >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() - {u'group1': {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']}, u'group2': {}} + OrderedDict([(u'group1', {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'5', u'5']}), (u'group2', {})]) >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5']} + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5'])]) >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() - {u'foo': [u"'bar'", u"'bazz'", u''], u'foo2': [u'2*5', u'6']} + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5', u'6'])]) >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() - {u'foo': [u"'bar'"]} + OrderedDict([(u'foo', [u"'bar'"])]) >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - {u'foo': [u"'bazz'"]} + OrderedDict([(u'foo', [u"'bazz'"])]) >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() - {u'foo': [u"'bar'"]} + OrderedDict([(u'foo', [u"'bar'"])]) + >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) + >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() + OrderedDict([(u'foo(2)', [u"'bar'"])]) + >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() + OrderedDict([(u'foo(2)', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) """ # Return empty dictionary for empty files. if self._len == 0: diff --git a/utils/python/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py similarity index 97% rename from utils/python/CIME/nmlgen.py rename to scripts/lib/CIME/nmlgen.py index 0b41c35effb..06dfd81e26f 100644 --- a/utils/python/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -96,6 +96,9 @@ def __exit__(self, *_): def init_defaults(self, infiles, config, skip_groups=None, skip_entry_loop=None ): """Return array of names of all definition nodes """ + # first clean out any settings left over from previous calls + self.new_instance() + self._definition.set_nodes(skip_groups=skip_groups) # Determine the array of entry nodes that will be acted upon @@ -211,7 +214,8 @@ def set_value(self, name, value): """ var_group = self._definition.get_group(name) literals = self._to_namelist_literals(name, value) - self._namelist.set_variable_value(var_group, name, literals) + _, _, var_size, = self._definition.split_type_string(name) + self._namelist.set_variable_value(var_group, name, literals, var_size) def get_default(self, name, config=None, allow_none=False): """Get the value of a variable from the namelist definition file. @@ -274,12 +278,24 @@ def get_default(self, name, config=None, allow_none=False): default[i] = self.quote_string(scalar) default = self._to_python_value(name, default) + return default def get_streams(self): """Get a list of all streams used for the current data model mode.""" return self.get_default("streamslist") + def clean_streams(self): + for variable in self._streams_variables: + self._streams_namelists[variable] = [] + self._streams_namelists["streams"] = [] + + def new_instance(self): + """ Clean the object just enough to introduce a new instance """ + self.clean_streams() + self._namelist.clean_groups() + + def _sub_fields(self, varnames): """Substitute indicators with given values in a list of fields. @@ -407,6 +423,7 @@ def create_stream_file_and_update_shr_strdata_nml(self, config, #pylint:disable= `stream_path` - Path to write the stream file to. `data_list_path` - Path of file to append input data information to. """ + # Stream-specific configuration. config = config.copy() config["stream"] = stream @@ -517,13 +534,8 @@ def add_default(self, name, value=None, ignore_abs_path=None): # Use this to see if we need to raise an error when nothing is found. have_value = False - # Check for existing value. current_literals = self._namelist.get_variable_value(group, name) - if current_literals != [""]: - have_value = True - # Do not proceed further since this has been obtained the -infile contents - return # Check for input argument. if value is not None: @@ -542,7 +554,8 @@ def add_default(self, name, value=None, ignore_abs_path=None): # Go through file names and prepend input data root directory for # absolute pathnames. - if ignore_abs_path is None: + var_type, _, var_size = self._definition.split_type_string(name) + if var_type == "character" and ignore_abs_path is None: var_input_pathname = self._definition.get_input_pathname(name) if var_input_pathname == 'abs': current_literals = expand_literal_list(current_literals) @@ -562,7 +575,7 @@ def add_default(self, name, value=None, ignore_abs_path=None): current_literals = compress_literal_list(current_literals) # Set the new value. - self._namelist.set_variable_value(group, name, current_literals) + self._namelist.set_variable_value(group, name, current_literals, var_size) def create_shr_strdata_nml(self): """Set defaults for `shr_strdata_nml` variables other than the variable domainfile """ diff --git a/utils/python/CIME/preview_namelists.py b/scripts/lib/CIME/preview_namelists.py similarity index 92% rename from utils/python/CIME/preview_namelists.py rename to scripts/lib/CIME/preview_namelists.py index 48b87549d13..3f1429809af 100644 --- a/utils/python/CIME/preview_namelists.py +++ b/scripts/lib/CIME/preview_namelists.py @@ -12,10 +12,10 @@ def create_dirs(case): Make necessary directories for case """ # Get data from XML - exeroot = case.get_value("EXEROOT") - libroot = case.get_value("LIBROOT") - incroot = case.get_value("INCROOT") - rundir = case.get_value("RUNDIR") + exeroot = case.get_value("EXEROOT") + libroot = case.get_value("LIBROOT") + incroot = case.get_value("INCROOT") + rundir = case.get_value("RUNDIR") caseroot = case.get_value("CASEROOT") docdir = os.path.join(caseroot, "CaseDocs") @@ -46,10 +46,7 @@ def create_namelists(case): """ case.flush() - # Needed in case user did case.build --clean-all - exeroot = case.get_value("EXEROOT") - if not os.path.exists(exeroot): - create_dirs(case) + create_dirs(case) casebuild = case.get_value("CASEBUILD") caseroot = case.get_value("CASEROOT") @@ -78,12 +75,14 @@ def create_namelists(case): cmd = os.path.join(config_dir, "buildnml") do_run_cmd = False + # This code will try to import and run each buildnml as a subroutine + # if that fails it will run it as a program in a seperate shell try: with open(cmd, 'r') as f: first_line = f.readline() if "python" in first_line: - logger.info(" Calling %s buildnml"%compname) mod = imp.load_source("buildnml", cmd) + logger.info(" Calling %s buildnml"%compname) mod.buildnml(case, caseroot, compname) else: raise SyntaxError diff --git a/utils/python/CIME/provenance.py b/scripts/lib/CIME/provenance.py similarity index 95% rename from utils/python/CIME/provenance.py rename to scripts/lib/CIME/provenance.py index 4da851dc4f5..26767495205 100644 --- a/utils/python/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -35,7 +35,7 @@ def save_build_provenance_acme(case, lid=None): describe_prov = os.path.join(exeroot, "GIT_DESCRIBE.%s" % lid) if os.path.exists(describe_prov): os.remove(describe_prov) - run_cmd_no_fail("git describe > %s" % describe_prov, from_dir=cimeroot) + run_cmd_no_fail("git describe", arg_stdout=describe_prov, from_dir=cimeroot) # Save HEAD headfile = os.path.join(cimeroot, ".git", "logs", "HEAD") @@ -119,12 +119,12 @@ def save_prerun_provenance_acme(case, lid=None): if mach == "mira": for cmd, filename in [("qstat -lf", "qstatf"), ("qstat -lf %s" % job_id, "qstatf_jobid")]: filename = "%s.%s" % (filename, lid) - run_cmd_no_fail("%s > %s" % (cmd, filename), from_dir=full_timing_dir) + run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename)) elif mach in ["edison", "cori-haswell", "cori-knl"]: for cmd, filename in [("sqs -f", "sqsf"), ("sqs -w -a", "sqsw"), ("sqs -f %s" % job_id, "sqsf_jobid"), ("squeue", "squeuef")]: filename = "%s.%s" % (filename, lid) - run_cmd_no_fail("%s > %s" % (cmd, filename), from_dir=full_timing_dir) + run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename)) elif mach == "titan": for cmd, filename in [("xtdb2proc -f", "xtdb2proc"), @@ -137,7 +137,7 @@ def save_prerun_provenance_acme(case, lid=None): gzip_existing_file(os.path.join(full_timing_dir, filename + "." + lid)) mdiag_reduce = os.path.join(full_timing_dir, "mdiag_reduce." + lid) - run_cmd_no_fail("./mdiag_reduce.csh > %s" % mdiag_reduce, from_dir=os.path.join(caseroot, "Tools")) + run_cmd_no_fail("./mdiag_reduce.csh", arg_stdout=mdiag_reduce, from_dir=os.path.join(caseroot, "Tools")) gzip_existing_file(mdiag_reduce) # copy/tar SourceModes @@ -190,9 +190,9 @@ def save_prerun_provenance_acme(case, lid=None): # Save state of repo if os.path.exists(os.path.join(cimeroot, ".git")): - run_cmd_no_fail("git describe > %s" % os.path.join(full_timing_dir, "GIT_DESCRIBE.%s" % lid), from_dir=cimeroot) + run_cmd_no_fail("git describe", arg_stdout=os.path.join(full_timing_dir, "GIT_DESCRIBE.%s" % lid), from_dir=cimeroot) else: - run_cmd_no_fail("git describe > %s" % os.path.join(full_timing_dir, "GIT_DESCRIBE.%s" % lid), from_dir=os.path.dirname(cimeroot)) + run_cmd_no_fail("git describe", arg_stdout=os.path.join(full_timing_dir, "GIT_DESCRIBE.%s" % lid), from_dir=os.path.dirname(cimeroot)) def save_prerun_provenance_cesm(case, lid=None): # pylint: disable=unused-argument pass diff --git a/utils/python/CIME/simple_compare.py b/scripts/lib/CIME/simple_compare.py similarity index 100% rename from utils/python/CIME/simple_compare.py rename to scripts/lib/CIME/simple_compare.py diff --git a/utils/python/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py similarity index 93% rename from utils/python/CIME/test_scheduler.py rename to scripts/lib/CIME/test_scheduler.py index 19471e0c95c..b3f977fc988 100644 --- a/utils/python/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -13,7 +13,7 @@ import CIME.compare_namelists import CIME.utils from update_acme_tests import get_recommended_test_time -from CIME.utils import append_status, TESTS_FAILED_ERR_CODE, parse_test_name, get_full_test_name, get_model +from CIME.utils import append_status, append_testlog, TESTS_FAILED_ERR_CODE, parse_test_name, get_full_test_name, get_model from CIME.test_status import * from CIME.XML.machines import Machines from CIME.XML.env_test import EnvTest @@ -87,15 +87,18 @@ def __init__(self, test_names, test_data=None, walltime=None, proc_pool=None, use_existing=False, save_timing=False, queue=None, allow_baseline_overwrite=False, output_root=None, - force_procs=None, force_threads=None, mpilib=None): + force_procs=None, force_threads=None, mpilib=None, input_dir=None): ########################################################################### - self._cime_root = CIME.utils.get_cime_root() - self._cime_model = get_model() - self._save_timing = save_timing - self._queue = queue - self._test_data = {} if test_data is None else test_data # Format: {test_name -> {data_name -> data}} - self._mpilib = mpilib # allow override of default mpilib - self._allow_baseline_overwrite = allow_baseline_overwrite + self._cime_root = CIME.utils.get_cime_root() + self._cime_model = get_model() + self._save_timing = save_timing + self._queue = queue + self._test_data = {} if test_data is None else test_data # Format: {test_name -> {data_name -> data}} + self._mpilib = mpilib # allow override of default mpilib + self._completed_tests = 0 + self._input_dir = input_dir + + self._allow_baseline_overwrite = allow_baseline_overwrite self._machobj = Machines(machine=machine_name) @@ -175,8 +178,8 @@ def __init__(self, test_names, test_data=None, if os.path.isdir(test_baseline): existing_baselines.append(test_baseline) expect(allow_baseline_overwrite or len(existing_baselines) == 0, - "Baseline directories already exists %s\n"\ - "Use --allow_baseline_overwrite to avoid this error"%existing_baselines) + "Baseline directories already exists %s\n" \ + "Use -o to avoid this error" % existing_baselines) else: self._baseline_root = None @@ -218,13 +221,15 @@ def __init__(self, test_names, test_data=None, else: self._update_test_status(test, phase, TEST_PEND_STATUS) self._update_test_status(test, phase, status) + logger.info("Using existing test directory %s"%(self._get_test_dir(test))) else: # None of the test directories should already exist. for test in self._tests: expect(not os.path.exists(self._get_test_dir(test)), "Cannot create new case in directory '%s', it already exists." " Pick a different test-id" % self._get_test_dir(test)) - logger.info("Created test in directory %s"%self._get_test_dir(test)) + logger.info("Creating test directory %s"%(self._get_test_dir(test))) + # By the end of this constructor, this program should never hard abort, # instead, errors will be placed in the TestStatus files for the various # tests cases @@ -237,7 +242,7 @@ def _log_output(self, test, output): # Note: making this directory could cause create_newcase to fail # if this is run before. os.makedirs(test_dir) - append_status(output,caseroot=test_dir,sfile="TestStatus.log") + append_testlog(output, caseroot=test_dir) ########################################################################### def _get_case_id(self, test): @@ -320,14 +325,14 @@ def _update_test_status(self, test, phase, status): def _shell_cmd_for_phase(self, test, cmd, phase, from_dir=None): ########################################################################### while True: - rc, output, errput = run_cmd(cmd, from_dir=from_dir) + rc, output, _ = run_cmd(cmd, combine_output=True, from_dir=from_dir) if rc != 0: self._log_output(test, - "%s FAILED for test '%s'.\nCommand: %s\nOutput: %s\n\nErrput: %s" % - (phase, test, cmd, output, errput)) + "%s FAILED for test '%s'.\nCommand: %s\nOutput: %s\n" % + (phase, test, cmd, output)) # Temporary hack to get around odd file descriptor use by # buildnml scripts. - if "bad interpreter" in errput: + if "bad interpreter" in output: time.sleep(1) continue else: @@ -335,10 +340,10 @@ def _shell_cmd_for_phase(self, test, cmd, phase, from_dir=None): else: # We don't want "RUN PASSED" in the TestStatus.log if the only thing that # succeeded was the submission. - if phase != RUN_PHASE or self._no_batch: - self._log_output(test, - "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s" % - (phase, test, cmd, output)) + phase = "SUBMIT" if phase == RUN_PHASE else phase + self._log_output(test, + "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s\n" % + (phase, test, cmd, output)) break return rc == 0 @@ -528,6 +533,9 @@ def _xml_phase(self, test): case.set_value("TEST", True) case.set_value("SAVE_TIMING", self._save_timing) + if self._input_dir is not None: + case.set_value("DIN_LOC_ROOT", self._input_dir) + return True ########################################################################### @@ -536,12 +544,9 @@ def _setup_phase(self, test): test_dir = self._get_test_dir(test) rv = self._shell_cmd_for_phase(test, "./case.setup", SETUP_PHASE, from_dir=test_dir) - # A little subtle. If namelists_only, the RUN phase, when the namelists would normally - # be handled, is not going to happen, so we have to do it here. - if self._namelists_only: - # It's OK for this command to fail with baseline diffs but not catastrophically - cmdstat, output, errput = run_cmd("./case.cmpgen_namelists", from_dir=test_dir) - expect(cmdstat in [0, TESTS_FAILED_ERR_CODE], "Fatal error in case.cmpgen_namelists: %s" % (output + "\n" + errput)) + # It's OK for this command to fail with baseline diffs but not catastrophically + cmdstat, output, _ = run_cmd("./case.cmpgen_namelists", combine_output=True, from_dir=test_dir) + expect(cmdstat in [0, TESTS_FAILED_ERR_CODE], "Fatal error in case.cmpgen_namelists: %s" % output) return rv @@ -644,8 +649,15 @@ def _consumer(self, test, test_phase, phase_method): if status != TEST_PEND_STATUS: self._update_test_status(test, test_phase, status) - status_str = "Finished %s for test %s in %f seconds (%s)" %\ - (test_phase, test, elapsed_time, status) + if not self._work_remains(test): + self._completed_tests += 1 + total = len(self._tests) + status_str = "Finished %s for test %s in %f seconds (%s). [COMPLETED %d of %d]" % \ + (test_phase, test, elapsed_time, status, self._completed_tests, total) + else: + status_str = "Finished %s for test %s in %f seconds (%s)" % \ + (test_phase, test, elapsed_time, status) + if not success: status_str += " Case dir: %s" % self._get_test_dir(test) logger.info(status_str) @@ -656,7 +668,7 @@ def _consumer(self, test, test_phase, phase_method): self._update_test_status_file(test, test_phase, status) if test_phase == XML_PHASE: - append_status("Case Created using: "+" ".join(sys.argv), caseroot=self._get_test_dir(test), sfile="README.case") + append_status("Case Created using: "+" ".join(sys.argv), "README.case", caseroot=self._get_test_dir(test)) # On batch systems, we want to immediately submit to the queue, because # it's very cheap to submit and will get us a better spot in line @@ -731,6 +743,8 @@ def _setup_cs_files(self): template = template.replace("", os.path.join(self._cime_root,"scripts","Tools")).replace\ ("", self._test_id) + if not os.path.exists(self._test_root): + os.makedirs(self._test_root) cs_status_file = os.path.join(self._test_root, "cs.status.%s" % self._test_id) with open(cs_status_file, "w") as fd: fd.write(template) @@ -777,13 +791,13 @@ def run_tests(self, wait=False): for test in self._tests: logger.info( " %s"% test) + # Setup cs files + self._setup_cs_files() + self._producer() expect(threading.active_count() == 1, "Leftover threads?") - # Setup cs files - self._setup_cs_files() - wait_handles_report = False if not self._no_run and not self._no_batch: if wait: @@ -816,7 +830,7 @@ def run_tests(self, wait=False): ts_status = ts.get_overall_test_status(ignore_namelists=True, check_memory=False, check_throughput=False) if ts_status not in [TEST_PASS_STATUS, TEST_PEND_STATUS]: - logger.info( "%s %s (phase %s)" % (status, test, phase)) + logger.info( "%s %s (phase %s)" % (ts_status, test, phase)) rv = False elif nlfail: logger.info( "%s %s (but otherwise OK) %s" % (NAMELIST_FAIL_STATUS, test, phase)) diff --git a/utils/python/CIME/test_status.py b/scripts/lib/CIME/test_status.py similarity index 98% rename from utils/python/CIME/test_status.py rename to scripts/lib/CIME/test_status.py index 3ab55d08b70..964b0dfd1b9 100644 --- a/utils/python/CIME/test_status.py +++ b/scripts/lib/CIME/test_status.py @@ -28,7 +28,6 @@ from collections import OrderedDict import os -from stat import S_IREAD, S_IRGRP, S_IROTH TEST_STATUS_FILENAME = "TestStatus" @@ -53,6 +52,7 @@ NAMELIST_PHASE = "NLCOMP" SHAREDLIB_BUILD_PHASE = "SHAREDLIB_BUILD" MODEL_BUILD_PHASE = "MODEL_BUILD" +SUBMIT_PHASE = "SUBMIT" RUN_PHASE = "RUN" THROUGHPUT_PHASE = "TPUTCOMP" MEMCOMP_PHASE = "MEMCOMP" @@ -67,6 +67,7 @@ NAMELIST_PHASE, SHAREDLIB_BUILD_PHASE, MODEL_BUILD_PHASE, + SUBMIT_PHASE, RUN_PHASE, COMPARE_PHASE, BASELINE_PHASE, @@ -81,6 +82,7 @@ SETUP_PHASE, SHAREDLIB_BUILD_PHASE, MODEL_BUILD_PHASE, + SUBMIT_PHASE, RUN_PHASE] def _test_helper1(file_contents): @@ -98,7 +100,7 @@ def _test_helper2(file_contents, wait_for_run=False, check_throughput=False, che class TestStatus(object): - def __init__(self, test_dir=None, test_name=None, no_io=False, lock=False): + def __init__(self, test_dir=None, test_name=None, no_io=False): """ Create a TestStatus object @@ -113,7 +115,6 @@ def __init__(self, test_dir=None, test_name=None, no_io=False, lock=False): self._test_name = test_name self._ok_to_modify = False self._no_io = no_io - self._lock = lock if os.path.exists(self._filename): self._parse_test_status_file() @@ -174,7 +175,7 @@ def set_status(self, phase, status, comments=""): if phase in CORE_PHASES and phase != CORE_PHASES[0]: previous_core_phase = CORE_PHASES[CORE_PHASES.index(phase)-1] - #TODO: enamble check below + #TODO: enable check below #expect(previous_core_phase in self._phase_statuses, "Core phase '%s' was skipped" % previous_core_phase) if previous_core_phase in self._phase_statuses: @@ -225,9 +226,6 @@ def flush(self): with open(self._filename, "w") as fd: self.phase_statuses_dump(fd) - if self._lock: - os.chmod(self._filename, S_IREAD | S_IRGRP | S_IROTH) - def _parse_test_status(self, file_contents): """ >>> contents = ''' diff --git a/utils/python/CIME/test_utils.py b/scripts/lib/CIME/test_utils.py similarity index 100% rename from utils/python/CIME/test_utils.py rename to scripts/lib/CIME/test_utils.py diff --git a/utils/python/CIME/tests/README b/scripts/lib/CIME/tests/README similarity index 100% rename from utils/python/CIME/tests/README rename to scripts/lib/CIME/tests/README diff --git a/utils/python/CIME/tests/SystemTests/__init__.py b/scripts/lib/CIME/tests/SystemTests/__init__.py similarity index 100% rename from utils/python/CIME/tests/SystemTests/__init__.py rename to scripts/lib/CIME/tests/SystemTests/__init__.py diff --git a/utils/python/CIME/tests/SystemTests/test_system_tests_compare_two.py b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py similarity index 100% rename from utils/python/CIME/tests/SystemTests/test_system_tests_compare_two.py rename to scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py diff --git a/utils/python/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py similarity index 100% rename from utils/python/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py rename to scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py diff --git a/utils/python/CIME/tests/SystemTests/test_utils/__init__.py b/scripts/lib/CIME/tests/SystemTests/test_utils/__init__.py similarity index 100% rename from utils/python/CIME/tests/SystemTests/test_utils/__init__.py rename to scripts/lib/CIME/tests/SystemTests/test_utils/__init__.py diff --git a/utils/python/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py b/scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py similarity index 100% rename from utils/python/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py rename to scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py diff --git a/utils/python/CIME/tests/__init__.py b/scripts/lib/CIME/tests/__init__.py similarity index 100% rename from utils/python/CIME/tests/__init__.py rename to scripts/lib/CIME/tests/__init__.py diff --git a/utils/python/CIME/tests/case_fake.py b/scripts/lib/CIME/tests/case_fake.py similarity index 100% rename from utils/python/CIME/tests/case_fake.py rename to scripts/lib/CIME/tests/case_fake.py diff --git a/utils/python/CIME/tests/test_case_fake.py b/scripts/lib/CIME/tests/test_case_fake.py similarity index 100% rename from utils/python/CIME/tests/test_case_fake.py rename to scripts/lib/CIME/tests/test_case_fake.py diff --git a/utils/python/CIME/user_mod_support.py b/scripts/lib/CIME/user_mod_support.py similarity index 100% rename from utils/python/CIME/user_mod_support.py rename to scripts/lib/CIME/user_mod_support.py diff --git a/utils/python/CIME/utils.py b/scripts/lib/CIME/utils.py similarity index 91% rename from utils/python/CIME/utils.py rename to scripts/lib/CIME/utils.py index 215fc9b8057..9216c85ff14 100644 --- a/utils/python/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -4,7 +4,7 @@ """ import logging, gzip, sys, os, time, re, shutil, glob, string, random import stat as statlib - +import warnings # Return this error code if the scripts worked but tests failed TESTS_FAILED_ERR_CODE = 100 logger = logging.getLogger(__name__) @@ -20,15 +20,44 @@ def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"): ... SystemExit: ERROR: error2 """ - if (not condition): + # Without this line we get a futurewarning on the use of condition below + warnings.filterwarnings("ignore") + if not condition: if logger.isEnabledFor(logging.DEBUG): import pdb pdb.set_trace() - raise exc_type("%s %s" % (error_prefix,error_msg)) + raise exc_type("%s %s"%(error_prefix,error_msg)) def id_generator(size=6, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) +def check_name(fullname, additional_chars=None, fullpath=False): + """ + check for unallowed characters in name, this routine only + checks the final name and does not check if path exists or is + writable + + >>> check_name("test.id", additional_chars=".") + False + >>> check_name("case.name", fullpath=False) + True + >>> check_name("/some/file/path/case.name", fullpath=True) + True + """ + + chars = '<>/{}[\]~`@' # pylint: disable=anomalous-backslash-in-string + if additional_chars is not None: + chars += additional_chars + if fullpath: + _, name = os.path.split(fullname) + else: + name = fullname + match = re.search(r"["+re.escape(chars)+"]", name) + if match is not None: + logger.warn("Illegal character %s found in name %s"%(match.group(0), name)) + return False + return True + # Should only be called from get_cime_config() def _read_cime_config_file(): """ @@ -69,7 +98,7 @@ def get_python_libs_location_within_cime(): """ From within CIME, return subdirectory of python libraries """ - return os.path.join("utils", "python") + return os.path.join("scripts", "lib") def get_cime_root(case=None): """ @@ -133,7 +162,7 @@ def get_model(): set_model(model) return model - modelroot = os.path.join(get_cime_root(), "cime_config") + modelroot = os.path.join(get_cime_root(), "config") models = os.listdir(modelroot) msg = ".cime/config or environment variable CIME_MODEL must be set to one of: " msg += ", ".join([model for model in models @@ -141,9 +170,15 @@ def get_model(): and model != "xml_schemas"]) expect(False, msg) +def _convert_to_fd(filearg, from_dir): + if not filearg.startswith("/") and from_dir is not None: + filearg = os.path.join(from_dir, filearg) + + return open(filearg, "a") + _hack=object() def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, - arg_stdout=_hack, arg_stderr=_hack, env=None): + arg_stdout=_hack, arg_stderr=_hack, env=None, combine_output=False): """ Wrapper around subprocess to make it much more convenient to run shell commands @@ -153,10 +188,15 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, import subprocess # Not safe to do globally, module not available in older pythons # Real defaults for these value should be subprocess.PIPE - if (arg_stdout is _hack): + if arg_stdout is _hack: arg_stdout = subprocess.PIPE - if (arg_stderr is _hack): - arg_stderr = subprocess.PIPE + elif isinstance(arg_stdout, str): + arg_stdout = _convert_to_fd(arg_stdout, from_dir) + + if arg_stderr is _hack: + arg_stderr = subprocess.STDOUT if combine_output else subprocess.PIPE + elif isinstance(arg_stderr, str): + arg_stderr = _convert_to_fd(arg_stdout, from_dir) if (verbose != False and (verbose or logger.isEnabledFor(logging.DEBUG))): logger.info("RUN: %s" % cmd) @@ -179,6 +219,12 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, errput = errput.strip() if errput is not None else errput stat = proc.wait() + if isinstance(arg_stdout, file): + arg_stdout.close() # pylint: disable=no-member + + if isinstance(arg_stderr, file) and arg_stderr is not arg_stdout: + arg_stderr.close() # pylint: disable=no-member + if (verbose != False and (verbose or logger.isEnabledFor(logging.DEBUG))): if stat != 0: logger.info(" stat: %d\n" % stat) @@ -190,7 +236,7 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None, return stat, output, errput def run_cmd_no_fail(cmd, input_str=None, from_dir=None, verbose=None, - arg_stdout=_hack, arg_stderr=_hack): + arg_stdout=_hack, arg_stderr=_hack, env=None, combine_output=False): """ Wrapper around subprocess to make it much more convenient to run shell commands. Expects command to work. Just returns output string. @@ -205,10 +251,18 @@ def run_cmd_no_fail(cmd, input_str=None, from_dir=None, verbose=None, >>> run_cmd_no_fail('grep foo', input_str='foo') 'foo' + + >>> run_cmd_no_fail('echo THE ERROR >&2', combine_output=True) + 'THE ERROR' """ - stat, output, errput = run_cmd(cmd, input_str, from_dir, verbose, arg_stdout, arg_stderr) - expect(stat == 0, "Command: '%s' failed with error '%s'%s" % - (cmd, errput, "" if from_dir is None else " from dir '%s'" % from_dir)) + stat, output, errput = run_cmd(cmd, input_str, from_dir, verbose, arg_stdout, arg_stderr, env, combine_output) + if stat != 0: + # If command produced no errput, put output in the exception since we + # have nothing else to go on. + errput = output if errput == "" else errput + expect(False, "Command: '%s' failed with error '%s'%s" % + (cmd, errput, "" if from_dir is None else " from dir '%s'" % from_dir)) + return output def check_minimum_python_version(major, minor): @@ -400,7 +454,7 @@ def get_cime_location_within_acme(): def get_model_config_location_within_cime(model=None): model = get_model() if model is None else model - return os.path.join("cime_config", model) + return os.path.join("config", model) def get_acme_root(): """ @@ -788,10 +842,10 @@ def format_time(time_format, input_format, input_time): >>> format_time("%H:%M:%S", "%H", "43") '43:00:00' - >>> format_time("%H %M", "%M.%S", "59,59") + >>> format_time("%H %M", "%M,%S", "59,59") '0 59' >>> format_time("%H, %S", "%H:%M:%S", "2:43:9") - '2, 09 + '2, 09' """ input_fields = input_format.split("%") expect(input_fields[0] == input_time[:len(input_fields[0])], @@ -847,17 +901,31 @@ def format_time(time_format, input_format, input_time): output_time += field[1:] return output_time -def append_status(msg, caseroot='.', sfile="CaseStatus"): +def append_status(msg, sfile, caseroot='.'): """ Append msg to sfile in caseroot """ - ctime = "" - # Don't put the time stamp in TestStatus - if sfile != "TestStatus": - ctime = time.strftime("%Y-%m-%d %H:%M:%S: ") - with open(os.path.join(caseroot,sfile), "a") as fd: - fd.write(ctime + msg + "\n") - fd.write("\n ---------------------------------------------------\n\n") + ctime = time.strftime("%Y-%m-%d %H:%M:%S: ") + + # Reduce empty lines in CaseStatus. It's a very concise file + # and does not need extra newlines for readability + line_ending = "" if sfile == "CaseStatus" else "\n" + + with open(os.path.join(caseroot, sfile), "a") as fd: + fd.write(ctime + msg + line_ending) + fd.write("\n ---------------------------------------------------\n" + line_ending) + +def append_testlog(msg, caseroot='.'): + """ + Add to TestStatus.log file + """ + append_status(msg, "TestStatus.log", caseroot) + +def append_case_status(phase, status, msg=None, caseroot='.'): + """ + Update CaseStatus file + """ + append_status("%s %s%s" % (phase, status, " %s" % msg if msg else ""), "CaseStatus", caseroot) def does_file_have_string(filepath, text): """ @@ -932,17 +1000,6 @@ def wait_for_unlocked(filepath): if file_object: file_object.close() -def get_build_threaded(case): - """Returns True if current settings require a threaded build/run.""" - force_threaded = case.get_value("BUILD_THREADED") - if force_threaded: - return True - comp_classes = case.get_values("COMP_CLASSES") - for comp_class in comp_classes: - if case.get_value("NTHRDS_%s"%comp_class) > 1: - return True - return False - def gunzip_existing_file(filepath): with gzip.open(filepath, "rb") as fd: return fd.read() @@ -1109,6 +1166,20 @@ def stringify_bool(val): expect(type(val) is bool, "Wrong type for val '%s'" % repr(val)) return "TRUE" if val else "FALSE" +def run_and_log_case_status(func, phase, caseroot='.'): + append_case_status(phase, "starting", caseroot=caseroot) + rv = None + try: + rv = func() + except: + e = sys.exc_info()[0] + append_case_status(phase, "error", msg=("\n%s" % e), caseroot=caseroot) + raise + else: + append_case_status(phase, "success", caseroot=caseroot) + + return rv + class SharedArea(object): """ Enable 0002 umask within this manager diff --git a/utils/python/CIME/wait_for_tests.py b/scripts/lib/CIME/wait_for_tests.py similarity index 99% rename from utils/python/CIME/wait_for_tests.py rename to scripts/lib/CIME/wait_for_tests.py index 1bbf6292994..efc850748d1 100644 --- a/utils/python/CIME/wait_for_tests.py +++ b/scripts/lib/CIME/wait_for_tests.py @@ -170,7 +170,7 @@ def create_cdash_upload_xml(results, cdash_build_name, cdash_build_group, utc_ti if (os.path.exists(tarball)): os.remove(tarball) - CIME.utils.run_cmd_no_fail("tar -cf - %s | gzip -c > %s" % (log_dir, tarball)) + CIME.utils.run_cmd_no_fail("tar -cf - %s | gzip -c" % log_dir, arg_stdout=tarball) base64 = CIME.utils.run_cmd_no_fail("base64 %s" % tarball) xml_text = \ diff --git a/utils/python/cs.status.template b/scripts/lib/cs.status.template similarity index 100% rename from utils/python/cs.status.template rename to scripts/lib/cs.status.template diff --git a/utils/python/cs.submit.template b/scripts/lib/cs.submit.template similarity index 100% rename from utils/python/cs.submit.template rename to scripts/lib/cs.submit.template diff --git a/utils/python/jenkins_generic_job.py b/scripts/lib/jenkins_generic_job.py similarity index 100% rename from utils/python/jenkins_generic_job.py rename to scripts/lib/jenkins_generic_job.py diff --git a/utils/python/update_acme_tests.py b/scripts/lib/update_acme_tests.py similarity index 100% rename from utils/python/update_acme_tests.py rename to scripts/lib/update_acme_tests.py diff --git a/scripts/manage_case b/scripts/manage_case index f4e9a9e0d14..c561d6cede4 100755 --- a/scripts/manage_case +++ b/scripts/manage_case @@ -131,9 +131,6 @@ def parse_command_line(args): parser.add_argument("--query-grids", action="store_true", help="Query supported model grids for %s model" %cime_model) - parser.add_argument("--grids-shortname", - help="Query model grids for input grid short name for %s model - not implemented yet" %cime_model) - parser.add_argument("--grids-alias", help="Query model grids for input grid alias for %s model - not implemented yet" %cime_model) diff --git a/scripts/manage_testlists b/scripts/manage_testlists index c8ad00c5f58..b077ec94899 100755 --- a/scripts/manage_testlists +++ b/scripts/manage_testlists @@ -326,10 +326,10 @@ sub writeXML my $newfilename; if ($component eq 'allactive') { - $newfilename = "$cimeroot/cime_config/${model}/allactive/testlist_allactive.xml"; + $newfilename = "$cimeroot/config/${model}/allactive/testlist_allactive.xml"; } elsif ($component eq 'drv') { - $newfilename = "$cimeroot/driver_cpl/cime_config/testdefs//testlist_drv.xml"; + $newfilename = "$cimeroot/src/drivers/mct/cime_config/testdefs//testlist_drv.xml"; } else { my $dir = "$cimeroot/../components/$component/cime_config/testdefs"; @@ -1073,7 +1073,7 @@ sub formattedOutput { "Compset", "TestName", "Grid", "Machine_compiler", "Test Category", "TestMods (optional)", "Comment(Optional)"); my %configcompsets; - my $xml = XML::LibXML->new()->parse_file("$cimeroot/cime_config/${model}/config_files.xml"); + my $xml = XML::LibXML->new()->parse_file("$cimeroot/config/${model}/config_files.xml"); my @nodes= $xml->findnodes(".//entry[\@id=\"COMPSETS_SPEC_FILE\"]/values/*"); foreach my $node (@nodes) { my $file = $node->textContent(); @@ -1187,9 +1187,9 @@ sub setTestlist{ my $list; if ($component =~ /^allactive/) { - $list = "$cimeroot/cime_config/${model}/allactive/testlist_allactive.xml"; + $list = "$cimeroot/config/${model}/allactive/testlist_allactive.xml"; } elsif ($component =~ /^drv/) { - $list = "$cimeroot/driver_cpl/cime_config/testdefs/testlist_drv.xml"; + $list = "$cimeroot/src/drivers/mct/cime_config/testdefs/testlist_drv.xml"; } else { $list = "$srcroot/components/$component/cime_config/testdefs/testlist_$component.xml"; } diff --git a/utils/python/tests/CMakeLists.txt b/scripts/tests/CMakeLists.txt similarity index 100% rename from utils/python/tests/CMakeLists.txt rename to scripts/tests/CMakeLists.txt diff --git a/utils/python/tests/CTestConfig.cmake b/scripts/tests/CTestConfig.cmake similarity index 100% rename from utils/python/tests/CTestConfig.cmake rename to scripts/tests/CTestConfig.cmake diff --git a/utils/python/tests/cpl.hi1.nc.test b/scripts/tests/cpl.hi1.nc.test similarity index 100% rename from utils/python/tests/cpl.hi1.nc.test rename to scripts/tests/cpl.hi1.nc.test diff --git a/utils/python/tests/cpl.hi2.nc.test b/scripts/tests/cpl.hi2.nc.test similarity index 100% rename from utils/python/tests/cpl.hi2.nc.test rename to scripts/tests/cpl.hi2.nc.test diff --git a/utils/python/tests/cpl.log.failmemleak.gz b/scripts/tests/cpl.log.failmemleak.gz similarity index 100% rename from utils/python/tests/cpl.log.failmemleak.gz rename to scripts/tests/cpl.log.failmemleak.gz diff --git a/utils/python/tests/cpl.log.passmemleak.gz b/scripts/tests/cpl.log.passmemleak.gz similarity index 100% rename from utils/python/tests/cpl.log.passmemleak.gz rename to scripts/tests/cpl.log.passmemleak.gz diff --git a/utils/python/tests/list_tests b/scripts/tests/list_tests similarity index 100% rename from utils/python/tests/list_tests rename to scripts/tests/list_tests diff --git a/utils/python/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py similarity index 93% rename from utils/python/tests/scripts_regression_tests.py rename to scripts/tests/scripts_regression_tests.py index 133caa2c176..762bf7bda70 100755 --- a/utils/python/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -5,7 +5,7 @@ from xml.etree.ElementTree import ParseError -LIB_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LIB_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),"..","lib") sys.path.append(LIB_DIR) # Remove all pyc files to ensure we're testing the right things import subprocess @@ -190,7 +190,7 @@ def test_a_unit_test(self): os.makedirs(test_dir) unit_test_tool = os.path.abspath(os.path.join(CIME.utils.get_cime_root(),"tools","unit_testing","run_tests.py")) test_spec_dir = os.path.join(os.path.dirname(unit_test_tool),"Examples", "interpolate_1d", "tests") - run_cmd_no_fail("%s --build-dir %s --test-spec-dir %s --compiler intel --use-openmp --mpirun-command mpirun.lsf"\ + run_cmd_no_fail("%s --build-dir %s --test-spec-dir %s"\ %(unit_test_tool,test_dir,test_spec_dir)) cls._do_teardown.append(test_dir) @@ -201,17 +201,18 @@ def test_b_cime_f90_unit_tests(self): machine = MACHINE.get_machine_name() compiler = MACHINE.get_default_compiler() - test_dir = os.path.join(cls._testroot,"driver_f90_tests") - cls._testdirs.append(test_dir) - os.makedirs(test_dir) if (machine != "yellowstone" or compiler != "intel"): #TODO: get rid of this restriction self.skipTest("Skipping TestUnitTest - only supported on yellowstone with intel") + + test_dir = os.path.join(cls._testroot,"driver_f90_tests") + cls._testdirs.append(test_dir) + os.makedirs(test_dir) test_spec_dir = CIME.utils.get_cime_root() unit_test_tool = os.path.abspath(os.path.join(test_spec_dir,"tools","unit_testing","run_tests.py")) - run_cmd_no_fail("%s --build-dir %s --test-spec-dir %s --compiler %s --use-openmp --mpirun-command mpirun.lsf"\ - %(unit_test_tool,test_dir,test_spec_dir, compiler)) + run_cmd_no_fail("%s --build-dir %s --test-spec-dir %s"\ + %(unit_test_tool,test_dir,test_spec_dir)) cls._do_teardown.append(test_dir) @classmethod @@ -226,10 +227,9 @@ def tearDownClass(cls): teardown_root = False elif do_teardown: shutil.rmtree(tfile) - if teardown_root: - shutil.rmtree(cls._testroot) - + if teardown_root and do_teardown: + shutil.rmtree(cls._testroot) ############################################################################### class J_TestCreateNewcase(unittest.TestCase): @@ -248,13 +248,32 @@ def test_a_createnewcase(self): shutil.rmtree(testdir) cls._testdirs.append(testdir) - run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16 --output-root %s" % (SCRIPT_DIR, testdir, cls._testroot), from_dir=SCRIPT_DIR) run_cmd_assert_result(self, "./case.setup", from_dir=testdir) run_cmd_assert_result(self, "./case.build", from_dir=testdir) cls._do_teardown.append(testdir) + def test_f_createnewcase_with_user_compset(self): + cls = self.__class__ + + testdir = os.path.join(cls._testroot, 'testcreatenewcase_with_user_compset') + if os.path.exists(testdir): + shutil.rmtree(testdir) + + cls._testdirs.append(testdir) + + pesfile = os.path.join("..","src","drivers","mct","cime_config","config_pes.xml") + args = "--case %s --compset 2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --user-compset --pesfile %s --res f19_g16 --output-root %s" % (testdir, pesfile, cls._testroot) + if CIME.utils.get_model() == "cesm": + args += " --run-unsupported" + + run_cmd_assert_result(self, "%s/create_newcase %s"%(SCRIPT_DIR, args), from_dir=SCRIPT_DIR) + run_cmd_assert_result(self, "./case.setup", from_dir=testdir) + run_cmd_assert_result(self, "./case.build", from_dir=testdir) + + cls._do_teardown.append(testdir) + def test_b_user_mods(self): cls = self.__class__ @@ -264,7 +283,7 @@ def test_b_user_mods(self): cls._testdirs.append(testdir) - user_mods_dir = os.path.join(CIME.utils.get_python_libs_root(), "tests", "user_mods_test1") + user_mods_dir = os.path.join(CIME.utils.get_python_libs_root(), "..", "tests", "user_mods_test1") run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16 --user-mods-dir %s --output-root %s" % (SCRIPT_DIR, testdir, user_mods_dir, cls._testroot),from_dir=SCRIPT_DIR) @@ -349,8 +368,10 @@ def test_e_xmlquery(self): output = run_cmd_no_fail(cmd, from_dir=casedir) self.assertTrue(output == JOB_QUEUE, msg="%s != %s"%(output, JOB_QUEUE)) - cls._do_teardown.append(cls._testroot) + cmd = xmlquery + " --listall" + run_cmd_no_fail(cmd, from_dir=casedir) + cls._do_teardown.append(cls._testroot) @classmethod def tearDownClass(cls): @@ -388,7 +409,7 @@ def setUp(self): make_fake_teststatus(os.path.join(self._testdir_with_fail, "5"), "Test_5", TEST_FAIL_STATUS, RUN_PHASE) make_fake_teststatus(os.path.join(self._testdir_unfinished, "5"), "Test_5", TEST_PEND_STATUS, RUN_PHASE) - make_fake_teststatus(os.path.join(self._testdir_unfinished2, "5"), "Test_5", TEST_PASS_STATUS, MODEL_BUILD_PHASE) + make_fake_teststatus(os.path.join(self._testdir_unfinished2, "5"), "Test_5", TEST_PASS_STATUS, SUBMIT_PHASE) # Set up proxy if possible self._unset_proxy = setup_proxy() @@ -761,9 +782,9 @@ def test_b_full(self): self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS) else: self.assertTrue(test_name in [pass_test, mem_pass_test]) - self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS) + self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS, msg=test_name) if (test_name == mem_pass_test): - self.assertEqual(ts.get_status(MEMLEAK_PHASE), TEST_PASS_STATUS) + self.assertEqual(ts.get_status(MEMLEAK_PHASE), TEST_PASS_STATUS, msg=test_name) ########################################################################### def test_c_use_existing(self): @@ -794,7 +815,7 @@ def test_c_use_existing(self): else: self.assertTrue(test_name == pass_test) self.assertEqual(ts.get_status(CIME.test_scheduler.MODEL_BUILD_PHASE), TEST_PASS_STATUS) - self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PEND_STATUS) + self.assertEqual(ts.get_status(CIME.test_scheduler.SUBMIT_PHASE), TEST_PEND_STATUS) os.environ["TESTBUILDFAIL_PASS"] = "True" ct2 = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH, use_existing=True, @@ -813,6 +834,7 @@ def test_c_use_existing(self): for test_status in test_statuses: ts = TestStatus(test_dir=os.path.dirname(test_status)) self.assertEqual(ts.get_status(CIME.test_scheduler.MODEL_BUILD_PHASE), TEST_PASS_STATUS) + self.assertEqual(ts.get_status(CIME.test_scheduler.SUBMIT_PHASE), TEST_PASS_STATUS) self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS) ############################################################################### @@ -909,6 +931,46 @@ def test_jenkins_generic_job_kill(self): self.assertTrue(self._thread_error is None, msg="Thread had failure: %s" % self._thread_error) assert_dashboard_has_build(self, build_name) +############################################################################### +class T_TestRunRestart(TestCreateTestCommon): +############################################################################### + + ########################################################################### + def test_run_restart(self): + ########################################################################### + run_cmd_assert_result(self, "%s/create_test --test-root %s --output-root %s -t %s NODEFAIL_P1.f45_g37.X" + % (SCRIPT_DIR, TEST_ROOT, TEST_ROOT, self._baseline_name)) + if self._hasbatch: + run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name), + from_dir=self._testroot) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name("NODEFAIL_P1.f45_g37.X", machine=self._machine, compiler=self._compiler), self._baseline_name)) + + fail_sentinel = os.path.join(casedir, "run", "FAIL_SENTINEL") + self.assertTrue(os.path.exists(fail_sentinel), msg="Missing %s" % fail_sentinel) + + self.assertEqual(open(fail_sentinel, "r").read().count("FAIL"), 3) + + ########################################################################### + def test_run_restart_too_many_fails(self): + ########################################################################### + os.environ["NODEFAIL_NUM_FAILS"] = "5" + run_cmd_assert_result(self, "%s/create_test --test-root %s --output-root %s -t %s NODEFAIL_P1.f45_g37.X" + % (SCRIPT_DIR, TEST_ROOT, TEST_ROOT, self._baseline_name), + expected_stat=(0 if self._hasbatch else CIME.utils.TESTS_FAILED_ERR_CODE)) + if self._hasbatch: + run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name), + from_dir=self._testroot, expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name("NODEFAIL_P1.f45_g37.X", machine=self._machine, compiler=self._compiler), self._baseline_name)) + + fail_sentinel = os.path.join(casedir, "run", "FAIL_SENTINEL") + self.assertTrue(os.path.exists(fail_sentinel), msg="Missing %s" % fail_sentinel) + + self.assertEqual(open(fail_sentinel, "r").read().count("FAIL"), 4) + ############################################################################### class Q_TestBlessTestResults(TestCreateTestCommon): ############################################################################### @@ -1203,7 +1265,7 @@ class K_TestCimeCase(TestCreateTestCommon): ########################################################################### def test_cime_case(self): ########################################################################### - run_cmd_assert_result(self, "%s/create_test cime_test_only -t %s --no-build --test-root %s --output-root %s" + run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s" % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT)) casedir = os.path.join(self._testroot, @@ -1239,6 +1301,45 @@ def test_cime_case(self): # Test some test properties self.assertEqual(case.get_value("TESTCASE"), "TESTRUNPASS") + ########################################################################### + def test_cime_case_build_threaded_1(self): + ########################################################################### + run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1x1.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s" + % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1x1.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + with Case(casedir, read_only=False) as case: + build_threaded = case.get_value("BUILD_THREADED") + self.assertFalse(build_threaded) + + build_threaded = case.get_build_threaded() + self.assertFalse(build_threaded) + + case.set_value("BUILD_THREADED", True) + + build_threaded = case.get_build_threaded() + self.assertTrue(build_threaded) + + ########################################################################### + def test_cime_case_build_threaded_2(self): + ########################################################################### + run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1x2.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s" + % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1x2.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + with Case(casedir, read_only=False) as case: + build_threaded = case.get_value("BUILD_THREADED") + self.assertFalse(build_threaded) + + build_threaded = case.get_build_threaded() + self.assertTrue(build_threaded) + ########################################################################### def test_cime_case_mpi_serial(self): ########################################################################### @@ -2104,7 +2205,17 @@ def _main_func(): expect(not hasattr(B_CheckCode, testname), "Repeat %s" % testname) setattr(B_CheckCode, testname, pylint_test) - unittest.main(verbosity=2, catchbreak=True) + try: + unittest.main(verbosity=2, catchbreak=True) + except SystemExit: + had_fails = sys.exc_info()[1].message + if had_fails: + print "Detected failures, leaving directory:", TEST_ROOT + else: + print "All pass, removing directory:", TEST_ROOT + if os.path.exists(TEST_ROOT): + shutil.rmtree(TEST_ROOT) + raise if (__name__ == "__main__"): _main_func() diff --git a/utils/python/tests/user_mods_test1/include_user_mods b/scripts/tests/user_mods_test1/include_user_mods similarity index 100% rename from utils/python/tests/user_mods_test1/include_user_mods rename to scripts/tests/user_mods_test1/include_user_mods diff --git a/utils/python/tests/user_mods_test1/user_nl_comp b/scripts/tests/user_mods_test1/user_nl_comp similarity index 100% rename from utils/python/tests/user_mods_test1/user_nl_comp rename to scripts/tests/user_mods_test1/user_nl_comp diff --git a/utils/python/tests/user_mods_test1/user_nl_cpl b/scripts/tests/user_mods_test1/user_nl_cpl similarity index 100% rename from utils/python/tests/user_mods_test1/user_nl_cpl rename to scripts/tests/user_mods_test1/user_nl_cpl diff --git a/utils/python/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 b/scripts/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 similarity index 100% rename from utils/python/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 rename to scripts/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 diff --git a/utils/python/tests/user_mods_test2/user_nl_cpl b/scripts/tests/user_mods_test2/user_nl_cpl similarity index 100% rename from utils/python/tests/user_mods_test2/user_nl_cpl rename to scripts/tests/user_mods_test2/user_nl_cpl diff --git a/share/csm_share/include/shr_assert.h b/share/csm_share/include/shr_assert.h deleted file mode 100644 index 8cbe490d175..00000000000 --- a/share/csm_share/include/shr_assert.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifdef NDEBUG -#define SHR_ASSERT(assert, msg) -#define SHR_ASSERT_ALL(assert, msg) -#define SHR_ASSERT_ANY(assert, msg) -#else -#define SHR_ASSERT(assert, msg) call shr_assert(assert, msg) -#define SHR_ASSERT_ALL(assert, msg) call shr_assert_all(assert, msg) -#define SHR_ASSERT_ANY(assert, msg) call shr_assert_any(assert, msg) -#endif -use shr_assert_mod diff --git a/share/csm_share/test/unit/shr_assert_test/test_macro.pf b/share/csm_share/test/unit/shr_assert_test/test_macro.pf deleted file mode 100644 index 87581d7a4d8..00000000000 --- a/share/csm_share/test/unit/shr_assert_test/test_macro.pf +++ /dev/null @@ -1,52 +0,0 @@ -module test_macro - -! Test that if NDEBUG is not defined, shr_assert macros run assertions. - -use pfunit_mod - -#undef NDEBUG -#include "shr_assert.h" - -contains - -@Test -subroutine macro_assert_can_pass() - SHR_ASSERT(.true., "Assert macro unexpectedly aborted!") -end subroutine macro_assert_can_pass - -@Test -subroutine macro_assert_can_fail() - SHR_ASSERT(.false., "Expected failure.") - ! When this was written, the preprocessor did not recognize this assert, - ! so call it directly instead of using an "@". - call assertExceptionRaised("ABORTED: Expected failure.") -end subroutine macro_assert_can_fail - -@Test -subroutine macro_assert_all_can_pass() - SHR_ASSERT_ALL(([.true., .true.]), "Assert macro unexpectedly aborted!") -end subroutine macro_assert_all_can_pass - -@Test -subroutine macro_assert_all_can_fail() - SHR_ASSERT_ALL(([.true., .false.]), "Expected failure.") - ! When this was written, the preprocessor did not recognize this assert, - ! so call it directly instead of using an "@". - call assertExceptionRaised("ABORTED: Expected failure.") -end subroutine macro_assert_all_can_fail - -@Test -subroutine macro_assert_any_can_pass() - SHR_ASSERT_ANY(([.true., .false.]), "Assert macro unexpectedly aborted!") -end subroutine macro_assert_any_can_pass - -@Test -subroutine macro_assert_any_can_fail() - SHR_ASSERT_ANY(([.false., .false.]), "Expected failure.") - ! When this was written, the preprocessor did not recognize this assert, - ! so call it directly instead of using an "@". - call assertExceptionRaised("ABORTED: Expected failure.") -end subroutine macro_assert_any_can_fail - - -end module test_macro diff --git a/share/csm_share/test/unit/shr_assert_test/test_ndebug.pf b/share/csm_share/test/unit/shr_assert_test/test_ndebug.pf deleted file mode 100644 index 9f90fcf5bc1..00000000000 --- a/share/csm_share/test/unit/shr_assert_test/test_ndebug.pf +++ /dev/null @@ -1,45 +0,0 @@ -module test_ndebug - -! Test that if NDEBUG is defined, shr_assert macros do nothing. - -use pfunit_mod - -#define NDEBUG -#include "shr_assert.h" - -contains - -@Test -subroutine ndebug_controls_assert_macro() - SHR_ASSERT(unreachable_function(), "Fake message.") -contains - logical function unreachable_function() - unreachable_function = .false. - call throw("NDEBUG failed to turn off SHR_ASSERT.", & - SourceLocation(__FILE__, __LINE__)) - end function unreachable_function -end subroutine ndebug_controls_assert_macro - -@Test -subroutine ndebug_controls_assert_all_macro() - SHR_ASSERT_ALL(unreachable_function(), "Fake message.") -contains - logical function unreachable_function() - unreachable_function = .false. - call throw("NDEBUG failed to turn off SHR_ASSERT_ALL.", & - SourceLocation(__FILE__, __LINE__)) - end function unreachable_function -end subroutine ndebug_controls_assert_all_macro - -@Test -subroutine ndebug_controls_assert_any_macro() - SHR_ASSERT_ANY(unreachable_function(), "Fake message.") -contains - logical function unreachable_function() - unreachable_function = .false. - call throw("NDEBUG failed to turn off SHR_ASSERT_ANY.", & - SourceLocation(__FILE__, __LINE__)) - end function unreachable_function -end subroutine ndebug_controls_assert_any_macro - -end module test_ndebug diff --git a/share/timing/COPYING b/share/timing/COPYING deleted file mode 100644 index 94a9ed024d3..00000000000 --- a/share/timing/COPYING +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/cime_config/buildlib.csm_share b/src/build_scripts/buildlib.csm_share similarity index 89% rename from cime_config/buildlib.csm_share rename to src/build_scripts/buildlib.csm_share index dd482bd91d7..4457a55ecac 100755 --- a/cime_config/buildlib.csm_share +++ b/src/build_scripts/buildlib.csm_share @@ -45,19 +45,19 @@ mkpath($libdir) unless -d $libdir; chdir($libdir) or die "Could not cd to $libdir: $!\n"; my @filepath = ( "$CASEROOT/SourceMods/src.share", - "$CIMEROOT/driver_cpl/shr", - "$CIMEROOT/components/xcpl_models/xshare", - "$CIMEROOT/share/csm_share/shr", - "$CIMEROOT/share/shr_RandNum/src", - "$CIMEROOT/share/shr_RandNum/src/dsfmt_f03", - "$CIMEROOT/share/shr_RandNum/src/kissvec", - "$CIMEROOT/share/shr_RandNum/src/mt19937", + "$CIMEROOT/src/drivers/mct/shr", + "$CIMEROOT/src/components/xcpl_models/xshare", + "$CIMEROOT/src/share/util", + "$CIMEROOT/src/share/RandNum/src", + "$CIMEROOT/src/share/RandNum/src/dsfmt_f03", + "$CIMEROOT/src/share/RandNum/src/kissvec", + "$CIMEROOT/src/share/RandNum/src/mt19937", ); if($useesmf eq "noesmf"){ - push(@filepath,"$CIMEROOT/share/esmf_wrf_timemgr"); + push(@filepath,"$CIMEROOT/src/share/esmf_wrf_timemgr"); }else{ - push(@filepath,"$CIMEROOT/driver_cpl/shr_esmf"); + push(@filepath,"$CIMEROOT/src/drivers/mct/shr_esmf"); } my @fp; diff --git a/cime_config/buildlib.gptl b/src/build_scripts/buildlib.gptl similarity index 90% rename from cime_config/buildlib.gptl rename to src/build_scripts/buildlib.gptl index 1269c3fd4cc..2a5fea6f852 100755 --- a/cime_config/buildlib.gptl +++ b/src/build_scripts/buildlib.gptl @@ -12,7 +12,7 @@ set GMAKE = `./xmlquery GMAKE -value ` # NOTE- (mv, 2015-01-02) SHAREDPATH is an environment variable set in # the $CASE.build script -setenv GPTL_DIR $CIMEROOT/share/timing # gptl directory +setenv GPTL_DIR $CIMEROOT/src/share/timing # gptl directory setenv GPTL_LIBDIR $1 setenv SHAREDPATH $2 diff --git a/cime_config/buildlib.mct b/src/build_scripts/buildlib.mct similarity index 95% rename from cime_config/buildlib.mct rename to src/build_scripts/buildlib.mct index c0bcc343fd2..9d29e86d285 100755 --- a/cime_config/buildlib.mct +++ b/src/build_scripts/buildlib.mct @@ -14,7 +14,7 @@ set MACH = `./xmlquery MACH -value ` set MPILIB = `./xmlquery MPILIB -value ` set OS = `./xmlquery OS -value ` -setenv MCT_DIR $CIMEROOT/externals/mct # mct directory +setenv MCT_DIR $CIMEROOT/src/externals/mct # mct directory setenv MCT_LIBDIR $1 setenv LIBDIR $2 echo "MCT_LIBDIR $MCT_LIBDIR" diff --git a/cime_config/buildlib.mpi-serial b/src/build_scripts/buildlib.mpi-serial similarity index 94% rename from cime_config/buildlib.mpi-serial rename to src/build_scripts/buildlib.mpi-serial index a065fc3eec0..1a68926a53f 100755 --- a/cime_config/buildlib.mpi-serial +++ b/src/build_scripts/buildlib.mpi-serial @@ -13,7 +13,7 @@ set GMAKE_J = `./xmlquery GMAKE_J -value ` set MACH = `./xmlquery MACH -value ` set OS = `./xmlquery OS -value ` -setenv MCT_DIR $CIMEROOT/externals/mct # mct directory +setenv MCT_DIR $CIMEROOT/src/externals/mct # mct directory setenv MCT_LIBDIR $1 setenv LIBDIR $2 diff --git a/cime_config/buildlib.pio b/src/build_scripts/buildlib.pio similarity index 92% rename from cime_config/buildlib.pio rename to src/build_scripts/buildlib.pio index 3556edc3a6e..cb4d29e40bf 100755 --- a/cime_config/buildlib.pio +++ b/src/build_scripts/buildlib.pio @@ -63,9 +63,7 @@ def _main_func(description): os.makedirs(pio_dir) os.chdir(pio_dir) casetools = case.get_value("CASETOOLS") - cmake_opts = "\"-D GENF90_PATH=$CIMEROOT/externals/genf90\ - -D CMAKE_MODULE_PATH=%s\" "%\ - os.path.join(CIME.utils.get_cime_root(), "externals","pio2","cmake") + cmake_opts = "\"-D GENF90_PATH=$CIMEROOT/src/externals/genf90 \"" gmake_opts = "%s/Makefile CASEROOT=%s MODEL=%s USER_CMAKE_OPTS=%s "\ "PIO_LIBDIR=%s CASETOOLS=%s PIO_VERSION=%s MPILIB=%s "\ "SHAREDLIBROOT=%s EXEROOT=%s COMPILER=%s BUILD_THREADED=%s "\ @@ -142,11 +140,20 @@ def _main_func(description): if netcdf4_parallel_found: valid_values += ",netcdf4p,netcdf4c" logger.warn("Updating valid_values for PIO_TYPENAME: %s", valid_values) + case.set_valid_values("PIO_TYPENAME",valid_values) # nothing means use the general default valid_values += ",nothing" - for comp in ("ATM","CPL","OCN","WAV","GLC","ICE","ROF","LND"): - case.set_valid_values("%s_PIO_TYPENAME"%comp,valid_values) + + for comp in case.get_values("COMP_CLASSES"): + comp_pio_typename = "%s_PIO_TYPENAME"%comp + case.set_valid_values(comp_pio_typename,valid_values) + current_value = case.get_value(comp_pio_typename) + if current_value not in valid_values: + logger.warn("Resetting PIO_TYPENAME to netcdf for component %s"%comp) + case.set_value(comp_pio_typename,"netcdf") + + if (__name__ == "__main__"): _main_func(__doc__) diff --git a/components/data_comps/datm/atm_comp_mct.F90 b/src/components/data_comps/datm/atm_comp_mct.F90 similarity index 100% rename from components/data_comps/datm/atm_comp_mct.F90 rename to src/components/data_comps/datm/atm_comp_mct.F90 diff --git a/components/data_comps/datm/cime_config/buildlib b/src/components/data_comps/datm/cime_config/buildlib similarity index 90% rename from components/data_comps/datm/cime_config/buildlib rename to src/components/data_comps/datm/cime_config/buildlib index 5f901d7dede..c8f9f39ec42 100755 --- a/components/data_comps/datm/cime_config/buildlib +++ b/src/components/data_comps/datm/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/datm/cime_config/buildnml b/src/components/data_comps/datm/cime_config/buildnml similarity index 98% rename from components/data_comps/datm/cime_config/buildnml rename to src/components/data_comps/datm/cime_config/buildnml index a4c49f3baa8..9b7751a0528 100755 --- a/components/data_comps/datm/cime_config/buildnml +++ b/src/components/data_comps/datm/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -147,11 +147,11 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # set variables that are not per-stream if datm_mode == 'CPLHISTForcingForOcnIce': - nmlgen.add_default("domainfile", value='null') if atm_domain_file != "UNSET": if case.get_value('ATM_DOMAIN_FILE') != 'UNSET': - case.add_default('ATM_DOMAIN_FILE', value='UNSET', ignore_abs_path=True) - case.flush() + nmlgen.add_default("domainfile", value=atm_domain_file, ignore_abs_path=True) + else: + nmlgen.add_default("domainfile", value='null') else: full_domain_path = os.path.join(atm_domain_path, atm_domain_file) nmlgen.add_default("domainfile", value=full_domain_path) @@ -203,7 +203,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/datm/cime_config/config_archive.xml b/src/components/data_comps/datm/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/datm/cime_config/config_archive.xml rename to src/components/data_comps/datm/cime_config/config_archive.xml diff --git a/components/data_comps/datm/cime_config/config_component.xml b/src/components/data_comps/datm/cime_config/config_component.xml similarity index 100% rename from components/data_comps/datm/cime_config/config_component.xml rename to src/components/data_comps/datm/cime_config/config_component.xml diff --git a/components/data_comps/datm/cime_config/namelist_definition_datm.xml b/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml similarity index 100% rename from components/data_comps/datm/cime_config/namelist_definition_datm.xml rename to src/components/data_comps/datm/cime_config/namelist_definition_datm.xml diff --git a/components/data_comps/datm/cime_config/user_nl_datm b/src/components/data_comps/datm/cime_config/user_nl_datm similarity index 100% rename from components/data_comps/datm/cime_config/user_nl_datm rename to src/components/data_comps/datm/cime_config/user_nl_datm diff --git a/components/data_comps/datm/datm_comp_mod.F90 b/src/components/data_comps/datm/datm_comp_mod.F90 similarity index 100% rename from components/data_comps/datm/datm_comp_mod.F90 rename to src/components/data_comps/datm/datm_comp_mod.F90 diff --git a/components/data_comps/datm/datm_shr_mod.F90 b/src/components/data_comps/datm/datm_shr_mod.F90 similarity index 100% rename from components/data_comps/datm/datm_shr_mod.F90 rename to src/components/data_comps/datm/datm_shr_mod.F90 diff --git a/components/data_comps/datm/doc/SOM.pdf b/src/components/data_comps/datm/doc/SOM.pdf similarity index 100% rename from components/data_comps/datm/doc/SOM.pdf rename to src/components/data_comps/datm/doc/SOM.pdf diff --git a/components/data_comps/datm/doc/bookinfo.xml b/src/components/data_comps/datm/doc/bookinfo.xml similarity index 100% rename from components/data_comps/datm/doc/bookinfo.xml rename to src/components/data_comps/datm/doc/bookinfo.xml diff --git a/components/data_comps/datm/doc/datacomps.xml b/src/components/data_comps/datm/doc/datacomps.xml similarity index 100% rename from components/data_comps/datm/doc/datacomps.xml rename to src/components/data_comps/datm/doc/datacomps.xml diff --git a/components/data_comps/datm/doc/intro.xml b/src/components/data_comps/datm/doc/intro.xml similarity index 100% rename from components/data_comps/datm/doc/intro.xml rename to src/components/data_comps/datm/doc/intro.xml diff --git a/components/data_comps/datm/doc/rundocbook.csh b/src/components/data_comps/datm/doc/rundocbook.csh similarity index 100% rename from components/data_comps/datm/doc/rundocbook.csh rename to src/components/data_comps/datm/doc/rundocbook.csh diff --git a/components/data_comps/datm/doc/streams.xml b/src/components/data_comps/datm/doc/streams.xml similarity index 99% rename from components/data_comps/datm/doc/streams.xml rename to src/components/data_comps/datm/doc/streams.xml index e3175863120..6d3ad3066bf 100644 --- a/components/data_comps/datm/doc/streams.xml +++ b/src/components/data_comps/datm/doc/streams.xml @@ -77,7 +77,7 @@ The strdata (short for "stream data") input is set via a fortran namelist called shr_strdata_nml. That namelist, the strdata datatype, and the methods are contained in the share source - code file, models/csm_share/shr/shr_strdata_mod.F90. In general, + code file, share/util/shr_strdata_mod.F90. In general, strdata input defines an array of input streams and operations to perform on those streams. Therefore, many namelist inputs are arrays of character strings. Different variable of the same index are diff --git a/components/data_comps/datm/doc/ug.xml b/src/components/data_comps/datm/doc/ug.xml similarity index 100% rename from components/data_comps/datm/doc/ug.xml rename to src/components/data_comps/datm/doc/ug.xml diff --git a/components/data_comps/desp/cime_config/buildlib b/src/components/data_comps/desp/cime_config/buildlib similarity index 90% rename from components/data_comps/desp/cime_config/buildlib rename to src/components/data_comps/desp/cime_config/buildlib index 5f9588e3652..682cbf335f5 100755 --- a/components/data_comps/desp/cime_config/buildlib +++ b/src/components/data_comps/desp/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/desp/cime_config/buildnml b/src/components/data_comps/desp/cime_config/buildnml similarity index 97% rename from components/data_comps/desp/cime_config/buildnml rename to src/components/data_comps/desp/cime_config/buildnml index 45ac5321569..6c938536441 100755 --- a/components/data_comps/desp/cime_config/buildnml +++ b/src/components/data_comps/desp/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -65,7 +65,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): config['desp_mode'] = desp_mode #---------------------------------------------------- - # Initialize namelist defaults + # Initialize namelist defaults #---------------------------------------------------- nmlgen.init_defaults(infile, config) @@ -99,7 +99,7 @@ def buildnml(case, caseroot, compname): os.makedirs(confdir) #---------------------------------------------------- - # Construct the namelist generator + # Construct the namelist generator #---------------------------------------------------- # Determine directory for user modified namelist_definitions.xml and namelist_defaults.xml user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname) @@ -107,7 +107,7 @@ def buildnml(case, caseroot, compname): "user_xml_dir %s does not exist " %user_xml_dir) # NOTE: User definition *replaces* existing definition. - namelist_xml_dir = os.path.join(cimeroot, "components", "data_comps", compname, "cime_config") + namelist_xml_dir = os.path.join(cimeroot, "src", "components", "data_comps", compname, "cime_config") definition_file = [os.path.join(namelist_xml_dir, "namelist_definition_desp.xml")] user_definition = os.path.join(user_xml_dir, "namelist_definition_desp.xml") if os.path.isfile(user_definition): @@ -117,12 +117,11 @@ def buildnml(case, caseroot, compname): # Create the namelist generator object - independent of instance nmlgen = NamelistGenerator(case, definition_file) - + #---------------------------------------------------- # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/desp/cime_config/config_component.xml b/src/components/data_comps/desp/cime_config/config_component.xml similarity index 100% rename from components/data_comps/desp/cime_config/config_component.xml rename to src/components/data_comps/desp/cime_config/config_component.xml diff --git a/components/data_comps/desp/cime_config/namelist_definition_desp.xml b/src/components/data_comps/desp/cime_config/namelist_definition_desp.xml similarity index 100% rename from components/data_comps/desp/cime_config/namelist_definition_desp.xml rename to src/components/data_comps/desp/cime_config/namelist_definition_desp.xml diff --git a/components/data_comps/desp/cime_config/user_nl_desp b/src/components/data_comps/desp/cime_config/user_nl_desp similarity index 100% rename from components/data_comps/desp/cime_config/user_nl_desp rename to src/components/data_comps/desp/cime_config/user_nl_desp diff --git a/components/data_comps/desp/desp_comp_mod.F90 b/src/components/data_comps/desp/desp_comp_mod.F90 similarity index 100% rename from components/data_comps/desp/desp_comp_mod.F90 rename to src/components/data_comps/desp/desp_comp_mod.F90 diff --git a/components/data_comps/desp/esp_comp_mct.F90 b/src/components/data_comps/desp/esp_comp_mct.F90 similarity index 100% rename from components/data_comps/desp/esp_comp_mct.F90 rename to src/components/data_comps/desp/esp_comp_mct.F90 diff --git a/components/data_comps/desp/esp_utils.F90 b/src/components/data_comps/desp/esp_utils.F90 similarity index 100% rename from components/data_comps/desp/esp_utils.F90 rename to src/components/data_comps/desp/esp_utils.F90 diff --git a/components/data_comps/dice/cime_config/buildlib b/src/components/data_comps/dice/cime_config/buildlib similarity index 90% rename from components/data_comps/dice/cime_config/buildlib rename to src/components/data_comps/dice/cime_config/buildlib index 4730f0ea102..02b3e57703a 100755 --- a/components/data_comps/dice/cime_config/buildlib +++ b/src/components/data_comps/dice/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/dice/cime_config/buildnml b/src/components/data_comps/dice/cime_config/buildnml similarity index 99% rename from components/data_comps/dice/cime_config/buildnml rename to src/components/data_comps/dice/cime_config/buildnml index e4d2be36b20..eef42ff3925 100755 --- a/components/data_comps/dice/cime_config/buildnml +++ b/src/components/data_comps/dice/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -164,7 +164,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/dice/cime_config/config_archive.xml b/src/components/data_comps/dice/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/dice/cime_config/config_archive.xml rename to src/components/data_comps/dice/cime_config/config_archive.xml diff --git a/components/data_comps/dice/cime_config/config_component.xml b/src/components/data_comps/dice/cime_config/config_component.xml similarity index 100% rename from components/data_comps/dice/cime_config/config_component.xml rename to src/components/data_comps/dice/cime_config/config_component.xml diff --git a/components/data_comps/dice/cime_config/namelist_definition_dice.xml b/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml similarity index 100% rename from components/data_comps/dice/cime_config/namelist_definition_dice.xml rename to src/components/data_comps/dice/cime_config/namelist_definition_dice.xml diff --git a/components/data_comps/dice/cime_config/user_nl_dice b/src/components/data_comps/dice/cime_config/user_nl_dice similarity index 100% rename from components/data_comps/dice/cime_config/user_nl_dice rename to src/components/data_comps/dice/cime_config/user_nl_dice diff --git a/components/data_comps/dice/dice_comp_mod.F90 b/src/components/data_comps/dice/dice_comp_mod.F90 similarity index 100% rename from components/data_comps/dice/dice_comp_mod.F90 rename to src/components/data_comps/dice/dice_comp_mod.F90 diff --git a/components/data_comps/dice/ice_comp_mct.F90 b/src/components/data_comps/dice/ice_comp_mct.F90 similarity index 100% rename from components/data_comps/dice/ice_comp_mct.F90 rename to src/components/data_comps/dice/ice_comp_mct.F90 diff --git a/components/data_comps/dlnd/cime_config/buildlib b/src/components/data_comps/dlnd/cime_config/buildlib similarity index 90% rename from components/data_comps/dlnd/cime_config/buildlib rename to src/components/data_comps/dlnd/cime_config/buildlib index 81bad4d7411..9f624fae95c 100755 --- a/components/data_comps/dlnd/cime_config/buildlib +++ b/src/components/data_comps/dlnd/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/dlnd/cime_config/buildnml b/src/components/data_comps/dlnd/cime_config/buildnml similarity index 99% rename from components/data_comps/dlnd/cime_config/buildnml rename to src/components/data_comps/dlnd/cime_config/buildnml index 04c6de1f9d6..09d88806eb7 100755 --- a/components/data_comps/dlnd/cime_config/buildnml +++ b/src/components/data_comps/dlnd/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -84,13 +84,11 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # Construct the list of streams. #---------------------------------------------------- streams = nmlgen.get_streams() - #---------------------------------------------------- # For each stream, create stream text file and update # shr_strdata_nml group and input data list. #---------------------------------------------------- for stream in streams: - # Ignore null values. if stream is None or stream in ("NULL", ""): continue @@ -108,7 +106,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): nmlgen.update_shr_strdata_nml(config, stream, stream_path) else: nmlgen.create_stream_file_and_update_shr_strdata_nml(config, stream, stream_path, data_list_path) - #---------------------------------------------------- # Create `shr_strdata_nml` namelist group. #---------------------------------------------------- @@ -162,12 +159,10 @@ def buildnml(case, caseroot, compname): # Create the namelist generator object - independent of instance nmlgen = NamelistGenerator(case, definition_file, files=files) - #---------------------------------------------------- # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/dlnd/cime_config/config_archive.xml b/src/components/data_comps/dlnd/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/dlnd/cime_config/config_archive.xml rename to src/components/data_comps/dlnd/cime_config/config_archive.xml diff --git a/components/data_comps/dlnd/cime_config/config_component.xml b/src/components/data_comps/dlnd/cime_config/config_component.xml similarity index 100% rename from components/data_comps/dlnd/cime_config/config_component.xml rename to src/components/data_comps/dlnd/cime_config/config_component.xml diff --git a/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml b/src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml similarity index 100% rename from components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml rename to src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml diff --git a/components/data_comps/dlnd/cime_config/user_nl_dlnd b/src/components/data_comps/dlnd/cime_config/user_nl_dlnd similarity index 100% rename from components/data_comps/dlnd/cime_config/user_nl_dlnd rename to src/components/data_comps/dlnd/cime_config/user_nl_dlnd diff --git a/components/data_comps/dlnd/dlnd_comp_mod.F90 b/src/components/data_comps/dlnd/dlnd_comp_mod.F90 similarity index 100% rename from components/data_comps/dlnd/dlnd_comp_mod.F90 rename to src/components/data_comps/dlnd/dlnd_comp_mod.F90 diff --git a/components/data_comps/dlnd/lnd_comp_mct.F90 b/src/components/data_comps/dlnd/lnd_comp_mct.F90 similarity index 100% rename from components/data_comps/dlnd/lnd_comp_mct.F90 rename to src/components/data_comps/dlnd/lnd_comp_mct.F90 diff --git a/components/data_comps/docn/cime_config/buildlib b/src/components/data_comps/docn/cime_config/buildlib similarity index 90% rename from components/data_comps/docn/cime_config/buildlib rename to src/components/data_comps/docn/cime_config/buildlib index 75ed5840ce4..1084a766a54 100755 --- a/components/data_comps/docn/cime_config/buildlib +++ b/src/components/data_comps/docn/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/docn/cime_config/buildnml b/src/components/data_comps/docn/cime_config/buildnml similarity index 99% rename from components/data_comps/docn/cime_config/buildnml rename to src/components/data_comps/docn/cime_config/buildnml index 42ec4c42417..2afb78283e5 100755 --- a/components/data_comps/docn/cime_config/buildnml +++ b/src/components/data_comps/docn/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -166,7 +166,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/docn/cime_config/config_archive.xml b/src/components/data_comps/docn/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/docn/cime_config/config_archive.xml rename to src/components/data_comps/docn/cime_config/config_archive.xml diff --git a/components/data_comps/docn/cime_config/config_component.xml b/src/components/data_comps/docn/cime_config/config_component.xml similarity index 87% rename from components/data_comps/docn/cime_config/config_component.xml rename to src/components/data_comps/docn/cime_config/config_component.xml index 335a75481aa..165d21c1dad 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/src/components/data_comps/docn/cime_config/config_component.xml @@ -113,6 +113,8 @@ $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_clim_pi_c101028.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc $DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc run_component_docn @@ -128,23 +130,28 @@ $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.fv0.9x1.25_gx1v7.151020.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc $DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.fv0.9x1.25_gx1v7.151020.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc $DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.fv0.9x1.25_gx1v7.151020.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.4x5_gx3v7_100120.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.4x5_gx3v7_100120.nc $DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc run_component_cam_sstice diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml similarity index 100% rename from components/data_comps/docn/cime_config/namelist_definition_docn.xml rename to src/components/data_comps/docn/cime_config/namelist_definition_docn.xml diff --git a/components/data_comps/docn/cime_config/user_nl_docn b/src/components/data_comps/docn/cime_config/user_nl_docn similarity index 100% rename from components/data_comps/docn/cime_config/user_nl_docn rename to src/components/data_comps/docn/cime_config/user_nl_docn diff --git a/components/data_comps/docn/docn_comp_mod.F90 b/src/components/data_comps/docn/docn_comp_mod.F90 similarity index 100% rename from components/data_comps/docn/docn_comp_mod.F90 rename to src/components/data_comps/docn/docn_comp_mod.F90 diff --git a/components/data_comps/docn/ocn_comp_mct.F90 b/src/components/data_comps/docn/ocn_comp_mct.F90 similarity index 100% rename from components/data_comps/docn/ocn_comp_mct.F90 rename to src/components/data_comps/docn/ocn_comp_mct.F90 diff --git a/components/data_comps/docn/tools/pop_som_frc/README b/src/components/data_comps/docn/tools/pop_som_frc/README similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/README rename to src/components/data_comps/docn/tools/pop_som_frc/README diff --git a/components/data_comps/docn/tools/pop_som_frc/SOM.doc b/src/components/data_comps/docn/tools/pop_som_frc/SOM.doc similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/SOM.doc rename to src/components/data_comps/docn/tools/pop_som_frc/SOM.doc diff --git a/components/data_comps/docn/tools/pop_som_frc/SOM.pdf b/src/components/data_comps/docn/tools/pop_som_frc/SOM.pdf similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/SOM.pdf rename to src/components/data_comps/docn/tools/pop_som_frc/SOM.pdf diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh b/src/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_frc.csh rename to src/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl b/src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl rename to src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl b/src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl rename to src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl b/src/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl rename to src/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl diff --git a/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh b/src/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh rename to src/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh diff --git a/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh b/src/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh rename to src/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh diff --git a/components/data_comps/drof/cime_config/buildlib b/src/components/data_comps/drof/cime_config/buildlib similarity index 90% rename from components/data_comps/drof/cime_config/buildlib rename to src/components/data_comps/drof/cime_config/buildlib index b53123f5dac..66bc96b84ae 100755 --- a/components/data_comps/drof/cime_config/buildlib +++ b/src/components/data_comps/drof/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/drof/cime_config/buildnml b/src/components/data_comps/drof/cime_config/buildnml similarity index 99% rename from components/data_comps/drof/cime_config/buildnml rename to src/components/data_comps/drof/cime_config/buildnml index e76eea5a8c3..37b0a3a7a3f 100755 --- a/components/data_comps/drof/cime_config/buildnml +++ b/src/components/data_comps/drof/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -164,7 +164,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/drof/cime_config/config_archive.xml b/src/components/data_comps/drof/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/drof/cime_config/config_archive.xml rename to src/components/data_comps/drof/cime_config/config_archive.xml diff --git a/components/data_comps/drof/cime_config/config_component.xml b/src/components/data_comps/drof/cime_config/config_component.xml similarity index 100% rename from components/data_comps/drof/cime_config/config_component.xml rename to src/components/data_comps/drof/cime_config/config_component.xml diff --git a/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml similarity index 100% rename from components/data_comps/drof/cime_config/namelist_definition_drof.xml rename to src/components/data_comps/drof/cime_config/namelist_definition_drof.xml diff --git a/components/data_comps/drof/cime_config/user_nl_drof b/src/components/data_comps/drof/cime_config/user_nl_drof similarity index 100% rename from components/data_comps/drof/cime_config/user_nl_drof rename to src/components/data_comps/drof/cime_config/user_nl_drof diff --git a/components/data_comps/drof/drof_comp_mod.F90 b/src/components/data_comps/drof/drof_comp_mod.F90 similarity index 100% rename from components/data_comps/drof/drof_comp_mod.F90 rename to src/components/data_comps/drof/drof_comp_mod.F90 diff --git a/components/data_comps/drof/rof_comp_mct.F90 b/src/components/data_comps/drof/rof_comp_mct.F90 similarity index 100% rename from components/data_comps/drof/rof_comp_mct.F90 rename to src/components/data_comps/drof/rof_comp_mct.F90 diff --git a/components/data_comps/dwav/README b/src/components/data_comps/dwav/README similarity index 100% rename from components/data_comps/dwav/README rename to src/components/data_comps/dwav/README diff --git a/components/data_comps/dwav/bld/README b/src/components/data_comps/dwav/bld/README similarity index 100% rename from components/data_comps/dwav/bld/README rename to src/components/data_comps/dwav/bld/README diff --git a/components/data_comps/dwav/cime_config/buildlib b/src/components/data_comps/dwav/cime_config/buildlib similarity index 90% rename from components/data_comps/dwav/cime_config/buildlib rename to src/components/data_comps/dwav/cime_config/buildlib index 1c9cb28ac5f..5d3b8e78b3f 100755 --- a/components/data_comps/dwav/cime_config/buildlib +++ b/src/components/data_comps/dwav/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/dwav/cime_config/buildnml b/src/components/data_comps/dwav/cime_config/buildnml similarity index 99% rename from components/data_comps/dwav/cime_config/buildnml rename to src/components/data_comps/dwav/cime_config/buildnml index 3a412278f2d..7a0d67cee2c 100755 --- a/components/data_comps/dwav/cime_config/buildnml +++ b/src/components/data_comps/dwav/cime_config/buildnml @@ -14,7 +14,7 @@ import shutil import sys import glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -170,7 +170,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/dwav/cime_config/config_archive.xml b/src/components/data_comps/dwav/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/dwav/cime_config/config_archive.xml rename to src/components/data_comps/dwav/cime_config/config_archive.xml diff --git a/components/data_comps/dwav/cime_config/config_component.xml b/src/components/data_comps/dwav/cime_config/config_component.xml similarity index 100% rename from components/data_comps/dwav/cime_config/config_component.xml rename to src/components/data_comps/dwav/cime_config/config_component.xml diff --git a/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml b/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml similarity index 100% rename from components/data_comps/dwav/cime_config/namelist_definition_dwav.xml rename to src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml diff --git a/components/data_comps/dwav/cime_config/user_nl_dwav b/src/components/data_comps/dwav/cime_config/user_nl_dwav similarity index 100% rename from components/data_comps/dwav/cime_config/user_nl_dwav rename to src/components/data_comps/dwav/cime_config/user_nl_dwav diff --git a/components/data_comps/dwav/dwav_comp_mod.F90 b/src/components/data_comps/dwav/dwav_comp_mod.F90 similarity index 100% rename from components/data_comps/dwav/dwav_comp_mod.F90 rename to src/components/data_comps/dwav/dwav_comp_mod.F90 diff --git a/components/data_comps/dwav/wav_comp_mct.F90 b/src/components/data_comps/dwav/wav_comp_mct.F90 similarity index 100% rename from components/data_comps/dwav/wav_comp_mct.F90 rename to src/components/data_comps/dwav/wav_comp_mct.F90 diff --git a/components/stub_comps/satm/cime_config/buildlib b/src/components/stub_comps/satm/cime_config/buildlib similarity index 90% rename from components/stub_comps/satm/cime_config/buildlib rename to src/components/stub_comps/satm/cime_config/buildlib index 83e8d9a5b58..0da3c680e27 100755 --- a/components/stub_comps/satm/cime_config/buildlib +++ b/src/components/stub_comps/satm/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/satm/cime_config/buildnml b/src/components/stub_comps/satm/cime_config/buildnml similarity index 100% rename from components/stub_comps/satm/cime_config/buildnml rename to src/components/stub_comps/satm/cime_config/buildnml diff --git a/components/stub_comps/satm/cime_config/config_component.xml b/src/components/stub_comps/satm/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/satm/cime_config/config_component.xml rename to src/components/stub_comps/satm/cime_config/config_component.xml diff --git a/components/stub_comps/satm/cpl/atm_comp_mct.F90 b/src/components/stub_comps/satm/cpl/atm_comp_mct.F90 similarity index 100% rename from components/stub_comps/satm/cpl/atm_comp_mct.F90 rename to src/components/stub_comps/satm/cpl/atm_comp_mct.F90 diff --git a/components/stub_comps/sesp/cime_config/buildlib b/src/components/stub_comps/sesp/cime_config/buildlib similarity index 90% rename from components/stub_comps/sesp/cime_config/buildlib rename to src/components/stub_comps/sesp/cime_config/buildlib index c54e5d4cfc8..47b48f32b3b 100755 --- a/components/stub_comps/sesp/cime_config/buildlib +++ b/src/components/stub_comps/sesp/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/sesp/cime_config/buildnml b/src/components/stub_comps/sesp/cime_config/buildnml similarity index 100% rename from components/stub_comps/sesp/cime_config/buildnml rename to src/components/stub_comps/sesp/cime_config/buildnml diff --git a/components/stub_comps/sesp/cime_config/config_component.xml b/src/components/stub_comps/sesp/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/sesp/cime_config/config_component.xml rename to src/components/stub_comps/sesp/cime_config/config_component.xml diff --git a/components/stub_comps/sesp/cpl/esp_comp_mct.F90 b/src/components/stub_comps/sesp/cpl/esp_comp_mct.F90 similarity index 100% rename from components/stub_comps/sesp/cpl/esp_comp_mct.F90 rename to src/components/stub_comps/sesp/cpl/esp_comp_mct.F90 diff --git a/components/stub_comps/sglc/cime_config/buildlib b/src/components/stub_comps/sglc/cime_config/buildlib similarity index 90% rename from components/stub_comps/sglc/cime_config/buildlib rename to src/components/stub_comps/sglc/cime_config/buildlib index dfaa11d6d12..7615fefee80 100755 --- a/components/stub_comps/sglc/cime_config/buildlib +++ b/src/components/stub_comps/sglc/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/sglc/cime_config/buildnml b/src/components/stub_comps/sglc/cime_config/buildnml similarity index 100% rename from components/stub_comps/sglc/cime_config/buildnml rename to src/components/stub_comps/sglc/cime_config/buildnml diff --git a/components/stub_comps/sglc/cime_config/config_component.xml b/src/components/stub_comps/sglc/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/sglc/cime_config/config_component.xml rename to src/components/stub_comps/sglc/cime_config/config_component.xml diff --git a/components/stub_comps/sglc/cpl/glc_comp_mct.F90 b/src/components/stub_comps/sglc/cpl/glc_comp_mct.F90 similarity index 100% rename from components/stub_comps/sglc/cpl/glc_comp_mct.F90 rename to src/components/stub_comps/sglc/cpl/glc_comp_mct.F90 diff --git a/components/stub_comps/sice/cime_config/buildlib b/src/components/stub_comps/sice/cime_config/buildlib similarity index 90% rename from components/stub_comps/sice/cime_config/buildlib rename to src/components/stub_comps/sice/cime_config/buildlib index a179856cc9a..6058bc7f6f7 100755 --- a/components/stub_comps/sice/cime_config/buildlib +++ b/src/components/stub_comps/sice/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/sice/cime_config/buildnml b/src/components/stub_comps/sice/cime_config/buildnml similarity index 100% rename from components/stub_comps/sice/cime_config/buildnml rename to src/components/stub_comps/sice/cime_config/buildnml diff --git a/components/stub_comps/sice/cime_config/config_component.xml b/src/components/stub_comps/sice/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/sice/cime_config/config_component.xml rename to src/components/stub_comps/sice/cime_config/config_component.xml diff --git a/components/stub_comps/sice/cpl/ice_comp_mct.F90 b/src/components/stub_comps/sice/cpl/ice_comp_mct.F90 similarity index 100% rename from components/stub_comps/sice/cpl/ice_comp_mct.F90 rename to src/components/stub_comps/sice/cpl/ice_comp_mct.F90 diff --git a/components/stub_comps/slnd/cime_config/buildlib b/src/components/stub_comps/slnd/cime_config/buildlib similarity index 90% rename from components/stub_comps/slnd/cime_config/buildlib rename to src/components/stub_comps/slnd/cime_config/buildlib index d464e7a1087..9bf3a0e5f8d 100755 --- a/components/stub_comps/slnd/cime_config/buildlib +++ b/src/components/stub_comps/slnd/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/slnd/cime_config/buildnml b/src/components/stub_comps/slnd/cime_config/buildnml similarity index 100% rename from components/stub_comps/slnd/cime_config/buildnml rename to src/components/stub_comps/slnd/cime_config/buildnml diff --git a/components/stub_comps/slnd/cime_config/config_component.xml b/src/components/stub_comps/slnd/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/slnd/cime_config/config_component.xml rename to src/components/stub_comps/slnd/cime_config/config_component.xml diff --git a/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 b/src/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 similarity index 100% rename from components/stub_comps/slnd/cpl/lnd_comp_mct.F90 rename to src/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 diff --git a/components/stub_comps/socn/cime_config/buildlib b/src/components/stub_comps/socn/cime_config/buildlib similarity index 90% rename from components/stub_comps/socn/cime_config/buildlib rename to src/components/stub_comps/socn/cime_config/buildlib index 559ea1c36ea..18ecb7480a4 100755 --- a/components/stub_comps/socn/cime_config/buildlib +++ b/src/components/stub_comps/socn/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/socn/cime_config/buildnml b/src/components/stub_comps/socn/cime_config/buildnml similarity index 100% rename from components/stub_comps/socn/cime_config/buildnml rename to src/components/stub_comps/socn/cime_config/buildnml diff --git a/components/stub_comps/socn/cime_config/config_component.xml b/src/components/stub_comps/socn/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/socn/cime_config/config_component.xml rename to src/components/stub_comps/socn/cime_config/config_component.xml diff --git a/components/stub_comps/socn/cpl/ocn_comp_mct.F90 b/src/components/stub_comps/socn/cpl/ocn_comp_mct.F90 similarity index 100% rename from components/stub_comps/socn/cpl/ocn_comp_mct.F90 rename to src/components/stub_comps/socn/cpl/ocn_comp_mct.F90 diff --git a/components/stub_comps/srof/cime_config/buildlib b/src/components/stub_comps/srof/cime_config/buildlib similarity index 90% rename from components/stub_comps/srof/cime_config/buildlib rename to src/components/stub_comps/srof/cime_config/buildlib index d448e16bcf8..7a3ad9192d6 100755 --- a/components/stub_comps/srof/cime_config/buildlib +++ b/src/components/stub_comps/srof/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/srof/cime_config/buildnml b/src/components/stub_comps/srof/cime_config/buildnml similarity index 100% rename from components/stub_comps/srof/cime_config/buildnml rename to src/components/stub_comps/srof/cime_config/buildnml diff --git a/components/stub_comps/srof/cime_config/config_component.xml b/src/components/stub_comps/srof/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/srof/cime_config/config_component.xml rename to src/components/stub_comps/srof/cime_config/config_component.xml diff --git a/components/stub_comps/srof/cpl/rof_comp_mct.F90 b/src/components/stub_comps/srof/cpl/rof_comp_mct.F90 similarity index 100% rename from components/stub_comps/srof/cpl/rof_comp_mct.F90 rename to src/components/stub_comps/srof/cpl/rof_comp_mct.F90 diff --git a/components/stub_comps/swav/cime_config/buildlib b/src/components/stub_comps/swav/cime_config/buildlib similarity index 90% rename from components/stub_comps/swav/cime_config/buildlib rename to src/components/stub_comps/swav/cime_config/buildlib index 0cb5ff5666a..04589b411ac 100755 --- a/components/stub_comps/swav/cime_config/buildlib +++ b/src/components/stub_comps/swav/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/swav/cime_config/buildnml b/src/components/stub_comps/swav/cime_config/buildnml similarity index 100% rename from components/stub_comps/swav/cime_config/buildnml rename to src/components/stub_comps/swav/cime_config/buildnml diff --git a/components/stub_comps/swav/cime_config/config_component.xml b/src/components/stub_comps/swav/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/swav/cime_config/config_component.xml rename to src/components/stub_comps/swav/cime_config/config_component.xml diff --git a/components/stub_comps/swav/cpl/wav_comp_mct.F90 b/src/components/stub_comps/swav/cpl/wav_comp_mct.F90 similarity index 100% rename from components/stub_comps/swav/cpl/wav_comp_mct.F90 rename to src/components/stub_comps/swav/cpl/wav_comp_mct.F90 diff --git a/components/xcpl_comps/xatm/cime_config/buildlib b/src/components/xcpl_comps/xatm/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xatm/cime_config/buildlib rename to src/components/xcpl_comps/xatm/cime_config/buildlib index 37fde2cc30d..594aa4a3459 100755 --- a/components/xcpl_comps/xatm/cime_config/buildlib +++ b/src/components/xcpl_comps/xatm/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xatm/cime_config/buildnml b/src/components/xcpl_comps/xatm/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xatm/cime_config/buildnml rename to src/components/xcpl_comps/xatm/cime_config/buildnml index b8104013501..50a814c79d3 100755 --- a/components/xcpl_comps/xatm/cime_config/buildnml +++ b/src/components/xcpl_comps/xatm/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xatm/cime_config/config_component.xml b/src/components/xcpl_comps/xatm/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xatm/cime_config/config_component.xml rename to src/components/xcpl_comps/xatm/cime_config/config_component.xml diff --git a/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 b/src/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 rename to src/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 diff --git a/components/xcpl_comps/xglc/cime_config/buildlib b/src/components/xcpl_comps/xglc/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xglc/cime_config/buildlib rename to src/components/xcpl_comps/xglc/cime_config/buildlib index 33fcc2187de..90e2e0afd43 100755 --- a/components/xcpl_comps/xglc/cime_config/buildlib +++ b/src/components/xcpl_comps/xglc/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xglc/cime_config/buildnml b/src/components/xcpl_comps/xglc/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xglc/cime_config/buildnml rename to src/components/xcpl_comps/xglc/cime_config/buildnml index a02ca3f951f..64077a469c3 100755 --- a/components/xcpl_comps/xglc/cime_config/buildnml +++ b/src/components/xcpl_comps/xglc/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xglc/cime_config/config_component.xml b/src/components/xcpl_comps/xglc/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xglc/cime_config/config_component.xml rename to src/components/xcpl_comps/xglc/cime_config/config_component.xml diff --git a/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 b/src/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 rename to src/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 diff --git a/components/xcpl_comps/xice/cime_config/buildlib b/src/components/xcpl_comps/xice/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xice/cime_config/buildlib rename to src/components/xcpl_comps/xice/cime_config/buildlib index 57137c68a9d..4b4157bd79e 100755 --- a/components/xcpl_comps/xice/cime_config/buildlib +++ b/src/components/xcpl_comps/xice/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xice/cime_config/buildnml b/src/components/xcpl_comps/xice/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xice/cime_config/buildnml rename to src/components/xcpl_comps/xice/cime_config/buildnml index f6fcbcc8a3b..e023dc7c6d2 100755 --- a/components/xcpl_comps/xice/cime_config/buildnml +++ b/src/components/xcpl_comps/xice/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xice/cime_config/config_component.xml b/src/components/xcpl_comps/xice/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xice/cime_config/config_component.xml rename to src/components/xcpl_comps/xice/cime_config/config_component.xml diff --git a/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 b/src/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xice/cpl/ice_comp_mct.F90 rename to src/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 diff --git a/components/xcpl_comps/xlnd/cime_config/buildlib b/src/components/xcpl_comps/xlnd/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xlnd/cime_config/buildlib rename to src/components/xcpl_comps/xlnd/cime_config/buildlib index 52d6dde1154..1f61fb8c23d 100755 --- a/components/xcpl_comps/xlnd/cime_config/buildlib +++ b/src/components/xcpl_comps/xlnd/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xlnd/cime_config/buildnml b/src/components/xcpl_comps/xlnd/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xlnd/cime_config/buildnml rename to src/components/xcpl_comps/xlnd/cime_config/buildnml index f2354a124ae..760d75e5c06 100755 --- a/components/xcpl_comps/xlnd/cime_config/buildnml +++ b/src/components/xcpl_comps/xlnd/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xlnd/cime_config/config_component.xml b/src/components/xcpl_comps/xlnd/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xlnd/cime_config/config_component.xml rename to src/components/xcpl_comps/xlnd/cime_config/config_component.xml diff --git a/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 b/src/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 rename to src/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 diff --git a/components/xcpl_comps/xocn/cime_config/buildlib b/src/components/xcpl_comps/xocn/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xocn/cime_config/buildlib rename to src/components/xcpl_comps/xocn/cime_config/buildlib index f376d7ae2ee..5a4aa3bfc21 100755 --- a/components/xcpl_comps/xocn/cime_config/buildlib +++ b/src/components/xcpl_comps/xocn/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xocn/cime_config/buildnml b/src/components/xcpl_comps/xocn/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xocn/cime_config/buildnml rename to src/components/xcpl_comps/xocn/cime_config/buildnml index 246cea26602..c6f9453766e 100755 --- a/components/xcpl_comps/xocn/cime_config/buildnml +++ b/src/components/xcpl_comps/xocn/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xocn/cime_config/config_component.xml b/src/components/xcpl_comps/xocn/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xocn/cime_config/config_component.xml rename to src/components/xcpl_comps/xocn/cime_config/config_component.xml diff --git a/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 b/src/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 rename to src/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 diff --git a/components/xcpl_comps/xrof/cime_config/buildlib b/src/components/xcpl_comps/xrof/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xrof/cime_config/buildlib rename to src/components/xcpl_comps/xrof/cime_config/buildlib index 3976c96932d..2245cae8312 100755 --- a/components/xcpl_comps/xrof/cime_config/buildlib +++ b/src/components/xcpl_comps/xrof/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xrof/cime_config/buildnml b/src/components/xcpl_comps/xrof/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xrof/cime_config/buildnml rename to src/components/xcpl_comps/xrof/cime_config/buildnml index 06e9f1b86bd..182cf39efa5 100755 --- a/components/xcpl_comps/xrof/cime_config/buildnml +++ b/src/components/xcpl_comps/xrof/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xrof/cime_config/config_component.xml b/src/components/xcpl_comps/xrof/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xrof/cime_config/config_component.xml rename to src/components/xcpl_comps/xrof/cime_config/config_component.xml diff --git a/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 b/src/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 rename to src/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 diff --git a/components/xcpl_comps/xshare/dead_data_mod.F90 b/src/components/xcpl_comps/xshare/dead_data_mod.F90 similarity index 100% rename from components/xcpl_comps/xshare/dead_data_mod.F90 rename to src/components/xcpl_comps/xshare/dead_data_mod.F90 diff --git a/components/xcpl_comps/xshare/dead_mct_mod.F90 b/src/components/xcpl_comps/xshare/dead_mct_mod.F90 similarity index 100% rename from components/xcpl_comps/xshare/dead_mct_mod.F90 rename to src/components/xcpl_comps/xshare/dead_mct_mod.F90 diff --git a/components/xcpl_comps/xshare/dead_mod.F90 b/src/components/xcpl_comps/xshare/dead_mod.F90 similarity index 100% rename from components/xcpl_comps/xshare/dead_mod.F90 rename to src/components/xcpl_comps/xshare/dead_mod.F90 diff --git a/components/xcpl_comps/xwav/cime_config/buildlib b/src/components/xcpl_comps/xwav/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xwav/cime_config/buildlib rename to src/components/xcpl_comps/xwav/cime_config/buildlib index df2c40dd11b..8ee3b413b33 100755 --- a/components/xcpl_comps/xwav/cime_config/buildlib +++ b/src/components/xcpl_comps/xwav/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xwav/cime_config/buildnml b/src/components/xcpl_comps/xwav/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xwav/cime_config/buildnml rename to src/components/xcpl_comps/xwav/cime_config/buildnml index ba0ae95a8a4..b7b8576babd 100755 --- a/components/xcpl_comps/xwav/cime_config/buildnml +++ b/src/components/xcpl_comps/xwav/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xwav/cime_config/config_component.xml b/src/components/xcpl_comps/xwav/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xwav/cime_config/config_component.xml rename to src/components/xcpl_comps/xwav/cime_config/config_component.xml diff --git a/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 b/src/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 rename to src/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 diff --git a/driver_cpl/bld/namelist_files/namelist_defaults_drv.xml b/src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_defaults_drv.xml rename to src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml diff --git a/driver_cpl/bld/namelist_files/namelist_definition_drv.xml b/src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_definition_drv.xml rename to src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml diff --git a/driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml b/src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml rename to src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml diff --git a/driver_cpl/bld/namelist_files/namelist_definition_modio.xml b/src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_definition_modio.xml rename to src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml diff --git a/driver_cpl/bld/testdir/Buildconf/camconf/drv_flds_in b/src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in similarity index 100% rename from driver_cpl/bld/testdir/Buildconf/camconf/drv_flds_in rename to src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in diff --git a/driver_cpl/bld/testdir/Buildconf/clmconf/drv_flds_in b/src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in similarity index 100% rename from driver_cpl/bld/testdir/Buildconf/clmconf/drv_flds_in rename to src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in diff --git a/driver_cpl/bld/testdir/env_build.xml b/src/drivers/mct/bld/testdir/env_build.xml similarity index 100% rename from driver_cpl/bld/testdir/env_build.xml rename to src/drivers/mct/bld/testdir/env_build.xml diff --git a/driver_cpl/bld/testdir/env_case.xml b/src/drivers/mct/bld/testdir/env_case.xml similarity index 100% rename from driver_cpl/bld/testdir/env_case.xml rename to src/drivers/mct/bld/testdir/env_case.xml diff --git a/driver_cpl/bld/testdir/env_mach_pes.xml b/src/drivers/mct/bld/testdir/env_mach_pes.xml similarity index 100% rename from driver_cpl/bld/testdir/env_mach_pes.xml rename to src/drivers/mct/bld/testdir/env_mach_pes.xml diff --git a/driver_cpl/bld/testdir/env_run.xml b/src/drivers/mct/bld/testdir/env_run.xml similarity index 99% rename from driver_cpl/bld/testdir/env_run.xml rename to src/drivers/mct/bld/testdir/env_run.xml index 1a1092f6f52..5711dcda702 100644 --- a/driver_cpl/bld/testdir/env_run.xml +++ b/src/drivers/mct/bld/testdir/env_run.xml @@ -1215,7 +1215,7 @@ char constant,diagnostic,prognostic - 1 + Determines how CLM will determine where CO2 is set. If value is constant, it will be set to CCSM_CO2_PPMV, if value is either diagnostic or prognostic, the atmosphere model diff --git a/driver_cpl/bld/testdir/runtest b/src/drivers/mct/bld/testdir/runtest similarity index 100% rename from driver_cpl/bld/testdir/runtest rename to src/drivers/mct/bld/testdir/runtest diff --git a/driver_cpl/bld/testdir/user_nl_cpl b/src/drivers/mct/bld/testdir/user_nl_cpl similarity index 100% rename from driver_cpl/bld/testdir/user_nl_cpl rename to src/drivers/mct/bld/testdir/user_nl_cpl diff --git a/driver_cpl/cime_config/buildexe b/src/drivers/mct/cime_config/buildexe similarity index 94% rename from driver_cpl/cime_config/buildexe rename to src/drivers/mct/cime_config/buildexe index 95c3d098e97..fbf07f1feb9 100755 --- a/driver_cpl/cime_config/buildexe +++ b/src/drivers/mct/cime_config/buildexe @@ -6,7 +6,7 @@ build model executable import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -35,7 +35,7 @@ def _main_func(): with open('Filepath', 'w') as out: out.write(os.path.join(caseroot, "SourceMods", "src.drv") + "\n") - out.write(os.path.join(cimeroot, "driver_cpl", "driver") + "\n") + out.write(os.path.join(cimeroot, "src", "drivers", "mct", "main") + "\n") # build model executable diff --git a/driver_cpl/cime_config/buildnml b/src/drivers/mct/cime_config/buildnml similarity index 90% rename from driver_cpl/cime_config/buildnml rename to src/drivers/mct/cime_config/buildnml index 4e29c3ff83c..71f68738141 100755 --- a/driver_cpl/cime_config/buildnml +++ b/src/drivers/mct/cime_config/buildnml @@ -7,9 +7,9 @@ # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position -import os, shutil, sys, glob +import os, shutil, sys, glob, itertools -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -190,15 +190,43 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): infiles.append(infile) if len(infiles) != 0: + + # First read the drv_flds_in files and make sure that + # for any key there are not two conflicting values + dicts = {} + for infile in infiles: + dict_ = {} + with open(infile) as myfile: + for line in myfile: + if "=" in line and '!' not in line: + name, var = line.partition("=")[::2] + name = name.strip() + var = var.strip() + dict_[name] = var + dicts[infile] = dict_ + + for first,second in itertools.combinations(dicts.keys(),2): + compare_drv_flds_in(dicts[first], dicts[second], first, second) + + # Now create drv_flds_in config = {} definition_file = [files.get_value("NAMELIST_DEFINITION_FILE", attribute={"component":"drv_flds"})] - nmlgen = NamelistGenerator(case, definition_file, files=files) skip_entry_loop = True nmlgen.init_defaults(infiles, config, skip_entry_loop=skip_entry_loop) drv_flds_in = os.path.join(caseroot, "CaseDocs", "drv_flds_in") nmlgen.write_output_file(drv_flds_in) +############################################################################### +def compare_drv_flds_in(first, second, infile1, infile2): +############################################################################### + sharedKeys = set(first.keys()).intersection(second.keys()) + for key in sharedKeys: + if first[key] != second[key]: + print('Key: {}, \n Value 1: {}, \n Value 2: {}'.format(key, first[key], second[key])) + expect(False, "incompatible settings in drv_flds_in from \n %s \n and \n %s" + % (infile1, infile2)) + ############################################################################### def _create_component_modelio_namelists(case, files): ############################################################################### @@ -290,12 +318,9 @@ def buildnml(case, caseroot, component): if case.get_value('COMP_ATM') == 'cam': # cam is actually changing the driver namelist settings cam_config_opts = case.get_value("CAM_CONFIG_OPTS") - if "adiabatic" in cam_config_opts: - infile_text = "atm_adiabatic = .true." - if "ideal" in cam_config_opts: - infile_text = "atm_ideal_phys = .true." if "aquaplanet" in cam_config_opts: infile_text = "aqua_planet = .true. \n aqua_planet_sst = 1" + user_nl_file = os.path.join(caseroot, "user_nl_cpl") namelist_infile = os.path.join(confdir, "namelist_infile") create_namelist_infile(case, user_nl_file, namelist_infile, infile_text) diff --git a/driver_cpl/cime_config/config_archive.xml b/src/drivers/mct/cime_config/config_archive.xml similarity index 100% rename from driver_cpl/cime_config/config_archive.xml rename to src/drivers/mct/cime_config/config_archive.xml diff --git a/driver_cpl/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml similarity index 97% rename from driver_cpl/cime_config/config_component.xml rename to src/drivers/mct/cime_config/config_component.xml index eeda67bca8a..6c655021b1e 100644 --- a/driver_cpl/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -935,6 +935,23 @@ this to work. + + char + ESMF_LOGKIND_SINGLE,ESMF_LOGKIND_MULTI,ESMF_LOGKIND_NONE + ESMF_LOGKIND_NONE + run_cesm + env_run.xml + + Determines what ESMF log files (if any) are generated when + USE_ESMF_LIB is TRUE. + ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from + all of the PETs. Not supported on some platforms. + ESMF_LOGKIND_MULTI: Use multiple log files -- one per PET. + ESMF_LOGKIND_NONE: Do not issue messages to a log file. + By default, no ESMF log files are generated. + + + logical TRUE,FALSE @@ -1804,6 +1821,16 @@ + + char + run_din + env_run.xml + + A regular expression to search for an indication that a run failure was caused by a node failure + and should therefore be re-attempted. + + + char UNSET @@ -1879,6 +1906,14 @@ + + integer + 0 + mach_pes + env_mach_pes.xml + Percent of extra spare nodes to allocate + + integer @@ -2477,6 +2512,23 @@ machines/compilers. + + + + char + + external_tools + env_run.xml + External script to be run before model completion + + + char + + external_tools + env_run.xml + External script to be run after model completion + + diff --git a/driver_cpl/cime_config/config_component_acme.xml b/src/drivers/mct/cime_config/config_component_acme.xml similarity index 100% rename from driver_cpl/cime_config/config_component_acme.xml rename to src/drivers/mct/cime_config/config_component_acme.xml diff --git a/driver_cpl/cime_config/config_component_cesm.xml b/src/drivers/mct/cime_config/config_component_cesm.xml similarity index 100% rename from driver_cpl/cime_config/config_component_cesm.xml rename to src/drivers/mct/cime_config/config_component_cesm.xml diff --git a/driver_cpl/cime_config/config_compsets.xml b/src/drivers/mct/cime_config/config_compsets.xml similarity index 100% rename from driver_cpl/cime_config/config_compsets.xml rename to src/drivers/mct/cime_config/config_compsets.xml diff --git a/driver_cpl/cime_config/config_pes.xml b/src/drivers/mct/cime_config/config_pes.xml similarity index 100% rename from driver_cpl/cime_config/config_pes.xml rename to src/drivers/mct/cime_config/config_pes.xml diff --git a/driver_cpl/cime_config/namelist_definition_drv.xml b/src/drivers/mct/cime_config/namelist_definition_drv.xml similarity index 99% rename from driver_cpl/cime_config/namelist_definition_drv.xml rename to src/drivers/mct/cime_config/namelist_definition_drv.xml index 844bcc65512..bffe785aef2 100644 --- a/driver_cpl/cime_config/namelist_definition_drv.xml +++ b/src/drivers/mct/cime_config/namelist_definition_drv.xml @@ -2602,6 +2602,24 @@ + + char + ccsm_pes + ccsm_pes + + Determines what ESMF log files (if any) are generated when + USE_ESMF_LIB is TRUE. + ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from + all of the PETs. Not supported on some platforms. + ESMF_LOGKIND_MULTI: Use multiple log files — one per PET. + ESMF_LOGKIND_NONE: Do not issue messages to a log file. + By default, no ESMF log files are generated. + + + $ESMF_LOGFILE_KIND + + + diff --git a/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml b/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml new file mode 100644 index 00000000000..1a9cc9376b8 --- /dev/null +++ b/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + char + abs + drv_flds_in + megan_emis_nl + + File containing MEGAN emissions factors. Includes the list of MEGAN compounds that can be + used in the Comp_Name variable on the file. + + + + + char(100) + drv_flds_in + megan_emis_nl + + MEGAN specifier. This is in the form of: Chem-compound = megan_compound(s) + where megan_compound(s) can be the sum of megan compounds with a "+" between them. + In each equation, the item to the left of the equal sign is a CAM chemistry compound, the + items to the right are compounds known to the MEGAN model (single or combinations). + For example: megan_specifier = 'ISOP = isoprene', 'C10H16 = pinene_a + carene_3 + thujene_a' + + + + + logical + drv_flds_in + megan_emis_nl + + MEGAN mapped isoprene emissions factors switch + If TRUE then use mapped MEGAN emissions factors for isoprene. + + + + + char(150) + drv_flds_in + drv_physics + + List of possible MEGAN compounds to use + (the list used by the simulation is on the megan_factors_file as the Comp_Name) + + + + + + + + + char + dry-deposition + drydep_inparm + xactive_lnd,xactive_atm,table + + Where dry deposition is calculated (from land, atmosphere, or from a table) + This specifies the method used to calculate dry + deposition velocities of gas-phase chemical species. The available methods are: + 'table' - prescribed method in CAM + 'xactive_atm' - interactive method in CAM + 'xactive_lnd' - interactive method in CLM + + + + + char(300) + dry-deposition + drydep_inparm + + List of species that undergo dry deposition. + + + + + + + + + char + abs + Fire_emissions + fire_emis_nl + + File containing fire emissions factors. + + + + + char(100) + Fire_emissions + fire_emis_nl + + Fire emissions specifier. + + + + + logical + Fire_emissions + fire_emis_nl + + If ture fire emissions are input into atmosphere as elevated forcings. + Otherwise they are treated as surface emissions. + + + + + + + + + char + carma + carma_inparm + + List of fluxes needed by the CARMA model, from CLM to CAM. + + + + diff --git a/driver_cpl/cime_config/namelist_definition_modelio.xml b/src/drivers/mct/cime_config/namelist_definition_modelio.xml similarity index 100% rename from driver_cpl/cime_config/namelist_definition_modelio.xml rename to src/drivers/mct/cime_config/namelist_definition_modelio.xml diff --git a/driver_cpl/cime_config/testdefs/testlist_drv.xml b/src/drivers/mct/cime_config/testdefs/testlist_drv.xml similarity index 100% rename from driver_cpl/cime_config/testdefs/testlist_drv.xml rename to src/drivers/mct/cime_config/testdefs/testlist_drv.xml diff --git a/driver_cpl/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands b/src/drivers/mct/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands similarity index 100% rename from driver_cpl/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands rename to src/drivers/mct/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands diff --git a/driver_cpl/cime_config/user_nl_cpl b/src/drivers/mct/cime_config/user_nl_cpl similarity index 100% rename from driver_cpl/cime_config/user_nl_cpl rename to src/drivers/mct/cime_config/user_nl_cpl diff --git a/driver_cpl/doc/bookinfo.xml b/src/drivers/mct/doc/bookinfo.xml similarity index 100% rename from driver_cpl/doc/bookinfo.xml rename to src/drivers/mct/doc/bookinfo.xml diff --git a/driver_cpl/doc/chap1.xml b/src/drivers/mct/doc/chap1.xml similarity index 99% rename from driver_cpl/doc/chap1.xml rename to src/drivers/mct/doc/chap1.xml index d302fa67b65..74356103095 100644 --- a/driver_cpl/doc/chap1.xml +++ b/src/drivers/mct/doc/chap1.xml @@ -20,8 +20,8 @@ and carries out mapping (interpolation), merging, diagnostics, and other calculations. The name cpl7 refers to the source code associated with both the driver and the coupler parts of the model. cpl7 code is located in the CIME source -tree under driver_cpl/ and the main program of CESM2 is -driver_cpl/driver/cesm_driver.F90. +tree under drivers/mct/ and the main program of CESM2 is +drivers/mct/main/cesm_driver.F90. This document provides a general overview of the cpl7 design. Then diff --git a/driver_cpl/doc/chap2.xml b/src/drivers/mct/doc/chap2.xml similarity index 100% rename from driver_cpl/doc/chap2.xml rename to src/drivers/mct/doc/chap2.xml diff --git a/driver_cpl/doc/figx1.jpg b/src/drivers/mct/doc/figx1.jpg similarity index 100% rename from driver_cpl/doc/figx1.jpg rename to src/drivers/mct/doc/figx1.jpg diff --git a/driver_cpl/doc/namelist.xml b/src/drivers/mct/doc/namelist.xml similarity index 100% rename from driver_cpl/doc/namelist.xml rename to src/drivers/mct/doc/namelist.xml diff --git a/driver_cpl/doc/rundocbook.csh b/src/drivers/mct/doc/rundocbook.csh similarity index 100% rename from driver_cpl/doc/rundocbook.csh rename to src/drivers/mct/doc/rundocbook.csh diff --git a/driver_cpl/doc/stylesheet.dsl b/src/drivers/mct/doc/stylesheet.dsl similarity index 100% rename from driver_cpl/doc/stylesheet.dsl rename to src/drivers/mct/doc/stylesheet.dsl diff --git a/driver_cpl/doc/ug.xml b/src/drivers/mct/doc/ug.xml similarity index 100% rename from driver_cpl/doc/ug.xml rename to src/drivers/mct/doc/ug.xml diff --git a/driver_cpl/driver/CMakeLists.txt b/src/drivers/mct/main/CMakeLists.txt similarity index 100% rename from driver_cpl/driver/CMakeLists.txt rename to src/drivers/mct/main/CMakeLists.txt diff --git a/driver_cpl/driver/cesm_comp_mod.F90 b/src/drivers/mct/main/cesm_comp_mod.F90 similarity index 100% rename from driver_cpl/driver/cesm_comp_mod.F90 rename to src/drivers/mct/main/cesm_comp_mod.F90 diff --git a/driver_cpl/driver/cesm_driver.F90 b/src/drivers/mct/main/cesm_driver.F90 similarity index 92% rename from driver_cpl/driver/cesm_driver.F90 rename to src/drivers/mct/main/cesm_driver.F90 index c846dea2887..7c22d4aa53c 100644 --- a/driver_cpl/driver/cesm_driver.F90 +++ b/src/drivers/mct/main/cesm_driver.F90 @@ -20,8 +20,9 @@ program cesm_driver !---------------------------------------------------------------------------- ! share code & libs !---------------------------------------------------------------------------- - use perf_mod - use ESMF + use perf_mod, only : t_startf, t_adj_detailf, t_stopf + use ESMF, only : ESMF_Initialize, ESMF_Finalize + use seq_comm_mct, only : esmf_logfile_kind use cesm_comp_mod, only : cesm_pre_init1 use cesm_comp_mod, only : cesm_pre_init2 use cesm_comp_mod, only : cesm_init @@ -40,7 +41,7 @@ program cesm_driver ! because it is needed for the time manager, even if the ESMF_INTERFACE ! is not used. !-------------------------------------------------------------------------- - call ESMF_Initialize() + call ESMF_Initialize(logkindflag=esmf_logfile_kind) !-------------------------------------------------------------------------- ! Read in the configuration information and initialize the time manager. diff --git a/driver_cpl/driver/component_mod.F90 b/src/drivers/mct/main/component_mod.F90 similarity index 100% rename from driver_cpl/driver/component_mod.F90 rename to src/drivers/mct/main/component_mod.F90 diff --git a/driver_cpl/driver/component_type_mod.F90 b/src/drivers/mct/main/component_type_mod.F90 similarity index 100% rename from driver_cpl/driver/component_type_mod.F90 rename to src/drivers/mct/main/component_type_mod.F90 diff --git a/driver_cpl/driver/cplcomp_exchange_mod.F90 b/src/drivers/mct/main/cplcomp_exchange_mod.F90 similarity index 100% rename from driver_cpl/driver/cplcomp_exchange_mod.F90 rename to src/drivers/mct/main/cplcomp_exchange_mod.F90 diff --git a/driver_cpl/driver/map_glc2lnd_mod.F90 b/src/drivers/mct/main/map_glc2lnd_mod.F90 similarity index 97% rename from driver_cpl/driver/map_glc2lnd_mod.F90 rename to src/drivers/mct/main/map_glc2lnd_mod.F90 index 4a8ab0d0389..9bb67b1de57 100644 --- a/driver_cpl/driver/map_glc2lnd_mod.F90 +++ b/src/drivers/mct/main/map_glc2lnd_mod.F90 @@ -248,7 +248,7 @@ subroutine get_glc_elevation_classes(glc_topo, glc_elevclass) !----------------------------------------------------------------------- npts = size(glc_elevclass) - SHR_ASSERT((size(glc_topo) == npts), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(glc_topo) == npts), __FILE__, __LINE__) do glc_pt = 1, npts call glc_get_elevation_class(glc_topo(glc_pt), glc_elevclass(glc_pt), err_code) @@ -291,8 +291,8 @@ subroutine get_frac_this_ec(glc_frac, glc_elevclass, this_elevclass, glc_frac_th !----------------------------------------------------------------------- npts = size(glc_frac_this_ec) - SHR_ASSERT((size(glc_frac) == npts), errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(glc_elevclass) == npts), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(glc_frac) == npts), __FILE__, __LINE__) + SHR_ASSERT_FL((size(glc_elevclass) == npts), __FILE__, __LINE__) if (this_elevclass == 0) then glc_frac_this_ec(:) = 1._r8 - glc_frac(:) @@ -340,7 +340,7 @@ subroutine set_topo_in_virtual_columns(elev_class, glc_frac_this_ec_l, & ! Extract fields from attribute vectors lsize = mct_aVect_lsize(glc_frac_this_ec_l) - SHR_ASSERT(mct_aVect_lsize(glc_topo_this_ec_l) == lsize, errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(mct_aVect_lsize(glc_topo_this_ec_l) == lsize, __FILE__, __LINE__) allocate(frac_l(lsize)) allocate(topo_l(lsize)) call mct_aVect_exportRattr(glc_frac_this_ec_l, frac_field, frac_l) @@ -388,7 +388,7 @@ subroutine make_aVect_frac_times_icemask(frac_av, mask_av, frac_field, icemask_f !----------------------------------------------------------------------- lsize = mct_aVect_lsize(frac_av) - SHR_ASSERT(mct_aVect_lsize(mask_av) == lsize, errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(mct_aVect_lsize(mask_av) == lsize, __FILE__, __LINE__) call mct_aVect_init(frac_times_icemask_av, rList = frac_times_icemask_field, lsize = lsize) call mct_aVect_copy(aVin = frac_av, aVout = frac_times_icemask_av, & diff --git a/driver_cpl/driver/map_lnd2glc_mod.F90 b/src/drivers/mct/main/map_lnd2glc_mod.F90 similarity index 98% rename from driver_cpl/driver/map_lnd2glc_mod.F90 rename to src/drivers/mct/main/map_lnd2glc_mod.F90 index 3cc273c5976..da2ea515880 100644 --- a/driver_cpl/driver/map_lnd2glc_mod.F90 +++ b/src/drivers/mct/main/map_lnd2glc_mod.F90 @@ -224,8 +224,8 @@ subroutine get_glc_elevation_classes(glc_ice_covered, glc_topo, glc_elevclass) !----------------------------------------------------------------------- npts = size(glc_elevclass) - SHR_ASSERT((size(glc_ice_covered) == npts), errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(glc_topo) == npts), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(glc_ice_covered) == npts), __FILE__, __LINE__) + SHR_ASSERT_FL((size(glc_topo) == npts), __FILE__, __LINE__) do glc_pt = 1, npts if (abs(glc_ice_covered(glc_pt) - 1._r8) < ice_covered_tol) then @@ -287,7 +287,7 @@ subroutine map_bare_land(l2x_l, landfrac_l, fieldname, mapper, data_g_bare_land) character(len=*), parameter :: subname = 'map_bare_land' !----------------------------------------------------------------------- - SHR_ASSERT(associated(data_g_bare_land), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(associated(data_g_bare_land), __FILE__, __LINE__) lsize_g = size(data_g_bare_land) elevclass_as_string = glc_elevclass_as_string(0) @@ -379,7 +379,7 @@ subroutine map_one_elevation_class(l2x_l, landfrac_l, fieldname, elevclass, & !----------------------------------------------------------------------- lsize_g = size(data_g_thisEC) - SHR_ASSERT((size(topo_g) == lsize_g), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(topo_g) == lsize_g), __FILE__, __LINE__) ! ------------------------------------------------------------------------ ! Create temporary attribute vectors diff --git a/driver_cpl/driver/map_lnd2rof_irrig_mod.F90 b/src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 similarity index 99% rename from driver_cpl/driver/map_lnd2rof_irrig_mod.F90 rename to src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 index eee6049c879..b7be99281fa 100644 --- a/driver_cpl/driver/map_lnd2rof_irrig_mod.F90 +++ b/src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 @@ -258,8 +258,8 @@ subroutine map_rof2lnd_volr(volr_r, mapper_Fr2l, volr_l) character(len=*), parameter :: volr_field = 'volr' !--------------------------------------------------------------- - SHR_ASSERT(associated(volr_r), errMsg(sourcefile, __LINE__)) - SHR_ASSERT(associated(volr_l), errMsg(sourcefile, __LINE__)) + SHR_ASSERT_FL(associated(volr_r), sourcefile, __LINE__) + SHR_ASSERT_FL(associated(volr_l), sourcefile, __LINE__) lsize_r = size(volr_r) lsize_l = size(volr_l) diff --git a/driver_cpl/driver/mrg_mod.F90 b/src/drivers/mct/main/mrg_mod.F90 similarity index 100% rename from driver_cpl/driver/mrg_mod.F90 rename to src/drivers/mct/main/mrg_mod.F90 diff --git a/driver_cpl/driver/prep_aoflux_mod.F90 b/src/drivers/mct/main/prep_aoflux_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_aoflux_mod.F90 rename to src/drivers/mct/main/prep_aoflux_mod.F90 diff --git a/driver_cpl/driver/prep_atm_mod.F90 b/src/drivers/mct/main/prep_atm_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_atm_mod.F90 rename to src/drivers/mct/main/prep_atm_mod.F90 diff --git a/driver_cpl/driver/prep_glc_mod.F90 b/src/drivers/mct/main/prep_glc_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_glc_mod.F90 rename to src/drivers/mct/main/prep_glc_mod.F90 diff --git a/driver_cpl/driver/prep_ice_mod.F90 b/src/drivers/mct/main/prep_ice_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_ice_mod.F90 rename to src/drivers/mct/main/prep_ice_mod.F90 diff --git a/driver_cpl/driver/prep_lnd_mod.F90 b/src/drivers/mct/main/prep_lnd_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_lnd_mod.F90 rename to src/drivers/mct/main/prep_lnd_mod.F90 diff --git a/driver_cpl/driver/prep_ocn_mod.F90 b/src/drivers/mct/main/prep_ocn_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_ocn_mod.F90 rename to src/drivers/mct/main/prep_ocn_mod.F90 diff --git a/driver_cpl/driver/prep_rof_mod.F90 b/src/drivers/mct/main/prep_rof_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_rof_mod.F90 rename to src/drivers/mct/main/prep_rof_mod.F90 diff --git a/driver_cpl/driver/prep_wav_mod.F90 b/src/drivers/mct/main/prep_wav_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_wav_mod.F90 rename to src/drivers/mct/main/prep_wav_mod.F90 diff --git a/driver_cpl/driver/seq_diag_mct.F90 b/src/drivers/mct/main/seq_diag_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_diag_mct.F90 rename to src/drivers/mct/main/seq_diag_mct.F90 diff --git a/driver_cpl/driver/seq_domain_mct.F90 b/src/drivers/mct/main/seq_domain_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_domain_mct.F90 rename to src/drivers/mct/main/seq_domain_mct.F90 diff --git a/driver_cpl/driver/seq_flux_mct.F90 b/src/drivers/mct/main/seq_flux_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_flux_mct.F90 rename to src/drivers/mct/main/seq_flux_mct.F90 diff --git a/driver_cpl/driver/seq_frac_mct.F90 b/src/drivers/mct/main/seq_frac_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_frac_mct.F90 rename to src/drivers/mct/main/seq_frac_mct.F90 diff --git a/driver_cpl/driver/seq_hist_mod.F90 b/src/drivers/mct/main/seq_hist_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_hist_mod.F90 rename to src/drivers/mct/main/seq_hist_mod.F90 diff --git a/driver_cpl/driver/seq_io_mod.F90 b/src/drivers/mct/main/seq_io_mod.F90 similarity index 99% rename from driver_cpl/driver/seq_io_mod.F90 rename to src/drivers/mct/main/seq_io_mod.F90 index 058e5160979..ad8649a6c11 100644 --- a/driver_cpl/driver/seq_io_mod.F90 +++ b/src/drivers/mct/main/seq_io_mod.F90 @@ -184,7 +184,9 @@ subroutine seq_io_wopen(filename,clobber,cdf64,file_ind) if (exists) then if (lclobber) then nmode = pio_clobber - if (lcdf64) nmode = ior(nmode,PIO_64BIT_OFFSET) + !lcdf64 only applies to classic NETCDF files. + if (lcdf64 .and. cpl_pio_iotype == PIO_IOTYPE_NETCDF) & + nmode = ior(nmode,PIO_64BIT_OFFSET) rcode = pio_createfile(cpl_io_subsystem, cpl_io_file(lfile_ind), cpl_pio_iotype, trim(filename), nmode) if(iam==0) write(logunit,*) subname,' create file ',trim(filename) rcode = pio_put_att(cpl_io_file(lfile_ind),pio_global,"file_version",version) @@ -203,7 +205,9 @@ subroutine seq_io_wopen(filename,clobber,cdf64,file_ind) endif else nmode = pio_noclobber - if (lcdf64) nmode = ior(nmode,PIO_64BIT_OFFSET) + !lcdf64 only applies to classic NETCDF files. + if (lcdf64 .and. cpl_pio_iotype == PIO_IOTYPE_NETCDF) & + nmode = ior(nmode,PIO_64BIT_OFFSET) rcode = pio_createfile(cpl_io_subsystem, cpl_io_file(lfile_ind), cpl_pio_iotype, trim(filename), nmode) if(iam==0) write(logunit,*) subname,' create file ',trim(filename) rcode = pio_put_att(cpl_io_file(lfile_ind),pio_global,"file_version",version) diff --git a/driver_cpl/driver/seq_map_mod.F90 b/src/drivers/mct/main/seq_map_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_map_mod.F90 rename to src/drivers/mct/main/seq_map_mod.F90 diff --git a/driver_cpl/driver/seq_map_type_mod.F90 b/src/drivers/mct/main/seq_map_type_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_map_type_mod.F90 rename to src/drivers/mct/main/seq_map_type_mod.F90 diff --git a/driver_cpl/driver/seq_rest_mod.F90 b/src/drivers/mct/main/seq_rest_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_rest_mod.F90 rename to src/drivers/mct/main/seq_rest_mod.F90 diff --git a/driver_cpl/driver/t_driver_timers_mod.F90 b/src/drivers/mct/main/t_driver_timers_mod.F90 similarity index 100% rename from driver_cpl/driver/t_driver_timers_mod.F90 rename to src/drivers/mct/main/t_driver_timers_mod.F90 diff --git a/driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 b/src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 similarity index 96% rename from driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 rename to src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 index b0a9cdfd5cb..e53a7894d1d 100644 --- a/driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 +++ b/src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 @@ -100,8 +100,8 @@ function constructor(field, topo, elevclass_bounds) result(this) this%num_points = size(field, 1) this%nelev = size(field, 2) - SHR_ASSERT_ALL((ubound(topo) == (/this%num_points, this%nelev/)), errMsg(__FILE__, __LINE__)) - SHR_ASSERT_ALL((ubound(elevclass_bounds) == (/this%nelev/)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL_FL((ubound(topo) == (/this%num_points, this%nelev/)), __FILE__, __LINE__) + SHR_ASSERT_ALL_FL((ubound(elevclass_bounds) == (/this%nelev/)), __FILE__, __LINE__) allocate(this%elevclass_bounds(0:this%nelev)) this%elevclass_bounds(:) = elevclass_bounds(:) @@ -231,8 +231,8 @@ subroutine get_gradients_one_class(this, elevation_class, gradients) ! Assert pre-conditions - SHR_ASSERT(this%calculated, errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(gradients) == this%num_points), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(this%calculated, __FILE__, __LINE__) + SHR_ASSERT_FL((size(gradients) == this%num_points), __FILE__, __LINE__) if (elevation_class < 1 .or. elevation_class > this%nelev) then write(logunit,*) subname, ': ERROR: elevation class out of bounds: ', & @@ -266,9 +266,9 @@ subroutine get_gradients_one_point(this, point, gradients) character(len=*), parameter :: subname = 'get_gradients_one_point' !----------------------------------------------------------------------- - SHR_ASSERT(this%calculated, errMsg(__FILE__, __LINE__)) - SHR_ASSERT(point <= this%num_points, errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(gradients) == this%nelev), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(this%calculated, __FILE__, __LINE__) + SHR_ASSERT_FL(point <= this%num_points, __FILE__, __LINE__) + SHR_ASSERT_FL((size(gradients) == this%nelev), __FILE__, __LINE__) gradients(:) = this%vertical_gradient(point, :) diff --git a/driver_cpl/driver/vertical_gradient_calculator_base.F90 b/src/drivers/mct/main/vertical_gradient_calculator_base.F90 similarity index 100% rename from driver_cpl/driver/vertical_gradient_calculator_base.F90 rename to src/drivers/mct/main/vertical_gradient_calculator_base.F90 diff --git a/driver_cpl/driver/vertical_gradient_calculator_factory.F90 b/src/drivers/mct/main/vertical_gradient_calculator_factory.F90 similarity index 98% rename from driver_cpl/driver/vertical_gradient_calculator_factory.F90 rename to src/drivers/mct/main/vertical_gradient_calculator_factory.F90 index 22455ce66cc..c6501a87e53 100644 --- a/driver_cpl/driver/vertical_gradient_calculator_factory.F90 +++ b/src/drivers/mct/main/vertical_gradient_calculator_factory.F90 @@ -55,7 +55,7 @@ function create_vertical_gradient_calculator_2nd_order( & !----------------------------------------------------------------------- nelev = size(elevclass_names) - SHR_ASSERT_ALL((ubound(elevclass_bounds) == (/nelev/)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL_FL((ubound(elevclass_bounds) == (/nelev/)), __FILE__, __LINE__) call extract_data_from_attr_vect(attr_vect, fieldname, toponame, elevclass_names, & field, topo) diff --git a/driver_cpl/shr/CMakeLists.txt b/src/drivers/mct/shr/CMakeLists.txt similarity index 100% rename from driver_cpl/shr/CMakeLists.txt rename to src/drivers/mct/shr/CMakeLists.txt diff --git a/driver_cpl/shr/glc_elevclass_mod.F90 b/src/drivers/mct/shr/glc_elevclass_mod.F90 similarity index 99% rename from driver_cpl/shr/glc_elevclass_mod.F90 rename to src/drivers/mct/shr/glc_elevclass_mod.F90 index def51ebecf3..8ae9ac652a5 100644 --- a/driver_cpl/shr/glc_elevclass_mod.F90 +++ b/src/drivers/mct/shr/glc_elevclass_mod.F90 @@ -126,7 +126,7 @@ subroutine glc_elevclass_init_override(my_glc_nec, my_topomax) character(len=*), parameter :: subname = 'glc_elevclass_init_override' !----------------------------------------------------------------------- - SHR_ASSERT_ALL((ubound(my_topomax) == (/my_glc_nec/)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL_FL((ubound(my_topomax) == (/my_glc_nec/)), __FILE__, __LINE__) glc_nec = my_glc_nec allocate(topomax(0:glc_nec)) diff --git a/driver_cpl/shr/seq_cdata_mod.F90 b/src/drivers/mct/shr/seq_cdata_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_cdata_mod.F90 rename to src/drivers/mct/shr/seq_cdata_mod.F90 diff --git a/driver_cpl/shr/seq_comm_mct.F90 b/src/drivers/mct/shr/seq_comm_mct.F90 similarity index 97% rename from driver_cpl/shr/seq_comm_mct.F90 rename to src/drivers/mct/shr/seq_comm_mct.F90 index 44dbd98f2ed..5062704cded 100644 --- a/driver_cpl/shr/seq_comm_mct.F90 +++ b/src/drivers/mct/shr/seq_comm_mct.F90 @@ -20,11 +20,13 @@ module seq_comm_mct use shr_sys_mod , only : shr_sys_abort, shr_sys_flush use shr_mpi_mod , only : shr_mpi_chkerr, shr_mpi_bcast, shr_mpi_max use shr_file_mod, only : shr_file_getUnit, shr_file_freeUnit + use esmf , only : ESMF_LogKind_Flag, ESMF_LOGKIND_NONE + use esmf , only : ESMF_LOGKIND_SINGLE, ESMF_LOGKIND_MULTI implicit none private -#include +#include save !-------------------------------------------------------------------------- @@ -145,7 +147,10 @@ module seq_comm_mct integer, public :: CPLWAVID(num_inst_wav) integer, public :: CPLESPID(num_inst_esp) + type(ESMF_LogKind_Flag), public :: esmf_logfile_kind + integer, parameter, public :: seq_comm_namelen=16 + type seq_comm_type character(len=seq_comm_namelen) :: name ! my name character(len=seq_comm_namelen) :: suffix ! recommended suffix @@ -182,19 +187,19 @@ module seq_comm_mct character(len=32), public :: & atm_layout, lnd_layout, ice_layout, glc_layout, rof_layout, & ocn_layout, wav_layout, esp_layout - + logical :: seq_comm_mct_initialized = .false. ! whether this module has been initialized !======================================================================= contains !====================================================================== - integer function seq_comm_get_ncomps() + integer function seq_comm_get_ncomps() seq_comm_get_ncomps = ncomps end function seq_comm_get_ncomps - + subroutine seq_comm_init(Comm_in, nmlfile) - + !---------------------------------------------------------- ! ! Arguments @@ -236,6 +241,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) ocn_ntasks, ocn_rootpe, ocn_pestride, ocn_nthreads, & esp_ntasks, esp_rootpe, esp_pestride, esp_nthreads, & cpl_ntasks, cpl_rootpe, cpl_pestride, cpl_nthreads + character(len=24) :: esmf_logging + namelist /ccsm_pes/ & atm_ntasks, atm_rootpe, atm_pestride, atm_nthreads, atm_layout, & lnd_ntasks, lnd_rootpe, lnd_pestride, lnd_nthreads, lnd_layout, & @@ -245,7 +252,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) rof_ntasks, rof_rootpe, rof_pestride, rof_nthreads, rof_layout, & ocn_ntasks, ocn_rootpe, ocn_pestride, ocn_nthreads, ocn_layout, & esp_ntasks, esp_rootpe, esp_pestride, esp_nthreads, esp_layout, & - cpl_ntasks, cpl_rootpe, cpl_pestride, cpl_nthreads + cpl_ntasks, cpl_rootpe, cpl_pestride, cpl_nthreads, esmf_logging !---------------------------------------------------------- ! make sure this is first pass and set comms unset @@ -264,7 +271,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) seq_comms(n)%inst = 0 seq_comms(n)%set = .false. seq_comms(n)%petlist_allocated = .false. - seq_comms(n)%mpicom = MPI_COMM_NULL ! do some initialization here + seq_comms(n)%mpicom = MPI_COMM_NULL ! do some initialization here seq_comms(n)%iam = -1 seq_comms(n)%iamroot = .false. seq_comms(n)%npes = -1 @@ -288,7 +295,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) ! Initialize gloiam on all IDs global_mype = mype - + do n = 1,ncomps seq_comms(n)%gloiam = mype enddo @@ -352,6 +359,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) cpl_pestride = 1 cpl_nthreads = 1 + esmf_logging = "ESMF_LOGKIND_NONE" + ! Read namelist if it exists nu = shr_file_getUnit() @@ -369,7 +378,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end if !--- compute some other num_inst values - + num_inst_xao = max(num_inst_atm,num_inst_ocn) num_inst_frc = num_inst_ice @@ -457,21 +466,21 @@ subroutine seq_comm_init(Comm_in, nmlfile) count = count + 1 CPLATMID(n) = count end do - + do n = 1, num_inst_lnd count = count + 1 LNDID(n) = count count = count + 1 CPLLNDID(n) = count end do - + do n = 1, num_inst_ocn count = count + 1 OCNID(n) = count count = count + 1 CPLOCNID(n) = count end do - + do n = 1, num_inst_ice count = count + 1 ICEID(n) = count @@ -526,7 +535,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) if (rof_rootpe < 0) error_state = .true. if (esp_rootpe < 0) error_state = .true. if (cpl_rootpe < 0) error_state = .true. - + if (error_state) then write(logunit,*) trim(subname),' ERROR: rootpes must be >= 0' call shr_sys_abort(trim(subname)//' ERROR: rootpes >= 0') @@ -577,7 +586,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Ocean instance tasks - + if (trim(ocn_layout) == trim(layout_concurrent)) then ocn_inst_tasks = ocn_ntasks / num_inst_ocn droot = (ocn_inst_tasks * ocn_pestride) @@ -597,7 +606,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Sea ice instance tasks - + if (trim(ice_layout) == trim(layout_concurrent)) then ice_inst_tasks = ice_ntasks / num_inst_ice droot = (ice_inst_tasks * ice_pestride) @@ -617,7 +626,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Glacier instance tasks - + if (trim(glc_layout) == trim(layout_concurrent)) then glc_inst_tasks = glc_ntasks / num_inst_glc droot = (glc_inst_tasks * glc_pestride) @@ -637,7 +646,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Runoff instance tasks - + if (trim(rof_layout) == trim(layout_concurrent)) then rof_inst_tasks = rof_ntasks / num_inst_rof droot = (rof_inst_tasks * rof_pestride) @@ -657,7 +666,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Wave instance tasks - + if (trim(wav_layout) == trim(layout_concurrent)) then wav_inst_tasks = wav_ntasks / num_inst_wav droot = (wav_inst_tasks * wav_pestride) @@ -677,7 +686,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! External System Processing instance tasks - + if (trim(esp_layout) == trim(layout_concurrent)) then esp_inst_tasks = esp_ntasks / num_inst_esp droot = (esp_inst_tasks * esp_pestride) @@ -890,6 +899,23 @@ subroutine seq_comm_init(Comm_in, nmlfile) deallocate(comps,comms) + ! ESMF logging (only has effect if ESMF libraries are used) + call mpi_bcast(esmf_logging, len(esmf_logging), MPI_CHARACTER, 0, GLOBAL_COMM, ierr) + + select case(esmf_logging) + case ("ESMF_LOGKIND_SINGLE") + esmf_logfile_kind = ESMF_LOGKIND_SINGLE + case ("ESMF_LOGKIND_MULTI") + esmf_logfile_kind = ESMF_LOGKIND_MULTI + case ("ESMF_LOGKIND_NONE") + esmf_logfile_kind = ESMF_LOGKIND_NONE + case default + if (mype == 0) then + write(logunit,*) trim(subname),' ERROR: Invalid value for esmf_logging, ',esmf_logging + endif + call shr_sys_abort(trim(subname)//' ERROR: Invalid value for esmf_logging '//esmf_logging) + end select + call seq_comm_printcomms() end subroutine seq_comm_init @@ -904,10 +930,10 @@ subroutine seq_comm_clean() ! seq_comm_init. integer :: id - + character(*), parameter :: subName = '(seq_comm_clean) ' !---------------------------------------------------------- - + if (.not. seq_comm_mct_initialized) then write(logunit,*) trim(subname),' ERROR seq_comm_init has not been called ' call shr_sys_abort() @@ -919,11 +945,11 @@ subroutine seq_comm_clean() deallocate(seq_comms(id)%petlist) end if end do - + call mct_world_clean() end subroutine seq_comm_clean - + !--------------------------------------------------------- subroutine seq_comm_setcomm(ID,pelist,nthreads,iname,inst,tinst) @@ -947,7 +973,7 @@ subroutine seq_comm_setcomm(ID,pelist,nthreads,iname,inst,tinst) if (ID < 1 .or. ID > ncomps) then write(logunit,*) subname,' ID out of range, abort ',ID call shr_sys_abort() - endif + endif call mpi_comm_group(GLOBAL_COMM, mpigrp_world, ierr) call shr_mpi_chkerr(ierr,subname//' mpi_comm_group mpigrp_world') @@ -1057,11 +1083,11 @@ subroutine seq_comm_joincomm(ID1,ID2,ID,iname,inst,tinst) if (ID1 < 1 .or. ID1 > ncomps) then write(logunit,*) subname,' ID1 out of range, abort ',ID1 call shr_sys_abort() - endif + endif if (ID2 < 1 .or. ID2 > ncomps) then write(logunit,*) subname,' ID2 out of range, abort ',ID2 call shr_sys_abort() - endif + endif if (ID < 1 .or. ID > ncomps) then write(logunit,*) subname,' ID out of range, abort ',ID call shr_sys_abort() @@ -1211,7 +1237,7 @@ subroutine seq_comm_jcommarr(IDs,ID,iname,inst,tinst) if (IDs(n) < 1 .or. IDs(n) > ncomps) then write(logunit,*) subname,' IDs out of range, abort ',n,IDs(n) call shr_sys_abort() - endif + endif if (.not. seq_comms(IDs(n))%set) then write(logunit,*) subname,' IDs not set ',n,IDs(n) call shr_sys_abort() @@ -1267,7 +1293,7 @@ subroutine seq_comm_jcommarr(IDs,ID,iname,inst,tinst) seq_comms(ID)%mpicom = mpicom seq_comms(ID)%mpigrp = mpigrp - + seq_comms(ID)%nthreads = 1 do n = 1,nids seq_comms(ID)%nthreads = max(seq_comms(ID)%nthreads,seq_comms(IDs(n))%nthreads) @@ -1355,7 +1381,7 @@ subroutine seq_comm_setptrs(ID,mpicom,mpigrp,npes,nthreads,iam,iamroot,gloiam,gl if ((ID == 0) .or. (ID > ncomps)) then write(logunit,*) subname,' ID out of range, return ',ID return - endif + endif if (present(mpicom)) then if (ID > 0) then diff --git a/driver_cpl/shr/seq_drydep_mod.F90 b/src/drivers/mct/shr/seq_drydep_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_drydep_mod.F90 rename to src/drivers/mct/shr/seq_drydep_mod.F90 diff --git a/driver_cpl/shr/seq_flds_mod.F90 b/src/drivers/mct/shr/seq_flds_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_flds_mod.F90 rename to src/drivers/mct/shr/seq_flds_mod.F90 diff --git a/driver_cpl/shr/seq_infodata_mod.F90 b/src/drivers/mct/shr/seq_infodata_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_infodata_mod.F90 rename to src/drivers/mct/shr/seq_infodata_mod.F90 diff --git a/driver_cpl/shr/seq_io_read_mod.F90 b/src/drivers/mct/shr/seq_io_read_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_io_read_mod.F90 rename to src/drivers/mct/shr/seq_io_read_mod.F90 diff --git a/driver_cpl/shr/seq_timemgr_mod.F90 b/src/drivers/mct/shr/seq_timemgr_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_timemgr_mod.F90 rename to src/drivers/mct/shr/seq_timemgr_mod.F90 diff --git a/driver_cpl/shr/shr_carma_mod.F90 b/src/drivers/mct/shr/shr_carma_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_carma_mod.F90 rename to src/drivers/mct/shr/shr_carma_mod.F90 diff --git a/driver_cpl/shr/shr_expr_parser_mod.F90 b/src/drivers/mct/shr/shr_expr_parser_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_expr_parser_mod.F90 rename to src/drivers/mct/shr/shr_expr_parser_mod.F90 diff --git a/driver_cpl/shr/shr_fire_emis_mod.F90 b/src/drivers/mct/shr/shr_fire_emis_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_fire_emis_mod.F90 rename to src/drivers/mct/shr/shr_fire_emis_mod.F90 diff --git a/driver_cpl/shr/shr_megan_mod.F90 b/src/drivers/mct/shr/shr_megan_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_megan_mod.F90 rename to src/drivers/mct/shr/shr_megan_mod.F90 diff --git a/driver_cpl/unit_test/CMakeLists.txt b/src/drivers/mct/unit_test/CMakeLists.txt similarity index 89% rename from driver_cpl/unit_test/CMakeLists.txt rename to src/drivers/mct/unit_test/CMakeLists.txt index 490759d1973..82603c37bfd 100644 --- a/driver_cpl/unit_test/CMakeLists.txt +++ b/src/drivers/mct/unit_test/CMakeLists.txt @@ -1,4 +1,4 @@ -set(DRV_ROOT "${CIME_ROOT}/driver_cpl") +set(DRV_ROOT "${CIME_ROOT}/src/drivers/mct") add_definitions( -DNUM_COMP_INST_ATM=1 @@ -14,17 +14,17 @@ add_definitions( # Add source directories from stubs. This should be done first, so that in the # case of name collisions, the drv versions take precedence (when there are two # files with the same name, the one added later wins). -add_subdirectory(${CIME_ROOT}/share/csm_share/unit_test_stubs/pio pio) +add_subdirectory(${CIME_ROOT}/src/share/unit_test_stubs/pio pio) # Add drv source directories add_subdirectory(${DRV_ROOT}/shr drv_shr) -add_subdirectory(${DRV_ROOT}/driver drv_driver) +add_subdirectory(${DRV_ROOT}/main drv_main) # Add general unit test directories (stubbed out files, etc.) add_subdirectory(utils drv_unit_test_utils) add_subdirectory(stubs drv_unit_test_stubs) -# Build libraries containing stuff needed for the unit tests. +# Build libraries containing stuff needed for the unit tests. # Eventually, these add_library calls should probably be distributed into the # correct location, rather than being in this top-level CMakeLists.txt file. # Note that we are including the stub pio in the csm_share library for simplicity. diff --git a/driver_cpl/unit_test/avect_wrapper_test/CMakeLists.txt b/src/drivers/mct/unit_test/avect_wrapper_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/avect_wrapper_test/CMakeLists.txt rename to src/drivers/mct/unit_test/avect_wrapper_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/avect_wrapper_test/test_avect_wrapper.pf b/src/drivers/mct/unit_test/avect_wrapper_test/test_avect_wrapper.pf similarity index 100% rename from driver_cpl/unit_test/avect_wrapper_test/test_avect_wrapper.pf rename to src/drivers/mct/unit_test/avect_wrapper_test/test_avect_wrapper.pf diff --git a/driver_cpl/unit_test/glc_elevclass_test/CMakeLists.txt b/src/drivers/mct/unit_test/glc_elevclass_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/glc_elevclass_test/CMakeLists.txt rename to src/drivers/mct/unit_test/glc_elevclass_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/glc_elevclass_test/test_glc_elevclass.pf b/src/drivers/mct/unit_test/glc_elevclass_test/test_glc_elevclass.pf similarity index 100% rename from driver_cpl/unit_test/glc_elevclass_test/test_glc_elevclass.pf rename to src/drivers/mct/unit_test/glc_elevclass_test/test_glc_elevclass.pf diff --git a/driver_cpl/unit_test/map_glc2lnd_test/CMakeLists.txt b/src/drivers/mct/unit_test/map_glc2lnd_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/map_glc2lnd_test/CMakeLists.txt rename to src/drivers/mct/unit_test/map_glc2lnd_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf b/src/drivers/mct/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf similarity index 100% rename from driver_cpl/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf rename to src/drivers/mct/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf diff --git a/driver_cpl/unit_test/map_lnd2glc_test/CMakeLists.txt b/src/drivers/mct/unit_test/map_lnd2glc_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/map_lnd2glc_test/CMakeLists.txt rename to src/drivers/mct/unit_test/map_lnd2glc_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf b/src/drivers/mct/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf similarity index 100% rename from driver_cpl/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf rename to src/drivers/mct/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf diff --git a/driver_cpl/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt b/src/drivers/mct/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt rename to src/drivers/mct/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf b/src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf similarity index 96% rename from driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf rename to src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf index a491d5e227b..f50bf8f3b1a 100644 --- a/driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf +++ b/src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf @@ -73,10 +73,10 @@ contains nlnd = map_l2r%get_n_source_points() nrof = map_l2r%get_n_dest_points() - call shr_assert(map_r2l%get_n_dest_points() == nlnd, errMsg(sourcefile, __LINE__)) - call shr_assert(map_r2l%get_n_source_points() == nrof, errMsg(sourcefile, __LINE__)) - call shr_assert(size(irrig_l) == nlnd, errMsg(sourcefile, __LINE__)) - call shr_assert(size(volr_r) == nrof, errMsg(sourcefile, __LINE__)) + call shr_assert(map_r2l%get_n_dest_points() == nlnd, file=sourcefile, line=__LINE__) + call shr_assert(map_r2l%get_n_source_points() == nrof, file=sourcefile, line=__LINE__) + call shr_assert(size(irrig_l) == nlnd, file=sourcefile, line=__LINE__) + call shr_assert(size(volr_r) == nrof, file=sourcefile, line=__LINE__) call create_aVect_with_data_rows_are_points(this%l2r_l, & attr_tags = [irrig_flux_field], & diff --git a/driver_cpl/unit_test/seq_map_test/CMakeLists.txt b/src/drivers/mct/unit_test/seq_map_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/seq_map_test/CMakeLists.txt rename to src/drivers/mct/unit_test/seq_map_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/seq_map_test/test_seq_map.pf b/src/drivers/mct/unit_test/seq_map_test/test_seq_map.pf similarity index 100% rename from driver_cpl/unit_test/seq_map_test/test_seq_map.pf rename to src/drivers/mct/unit_test/seq_map_test/test_seq_map.pf diff --git a/driver_cpl/unit_test/stubs/CMakeLists.txt b/src/drivers/mct/unit_test/stubs/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/stubs/CMakeLists.txt rename to src/drivers/mct/unit_test/stubs/CMakeLists.txt diff --git a/driver_cpl/unit_test/stubs/vertical_gradient_calculator_constant.F90 b/src/drivers/mct/unit_test/stubs/vertical_gradient_calculator_constant.F90 similarity index 100% rename from driver_cpl/unit_test/stubs/vertical_gradient_calculator_constant.F90 rename to src/drivers/mct/unit_test/stubs/vertical_gradient_calculator_constant.F90 diff --git a/driver_cpl/unit_test/utils/CMakeLists.txt b/src/drivers/mct/unit_test/utils/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/utils/CMakeLists.txt rename to src/drivers/mct/unit_test/utils/CMakeLists.txt diff --git a/driver_cpl/unit_test/utils/avect_wrapper_mod.F90 b/src/drivers/mct/unit_test/utils/avect_wrapper_mod.F90 similarity index 100% rename from driver_cpl/unit_test/utils/avect_wrapper_mod.F90 rename to src/drivers/mct/unit_test/utils/avect_wrapper_mod.F90 diff --git a/driver_cpl/unit_test/utils/create_mapper_mod.F90 b/src/drivers/mct/unit_test/utils/create_mapper_mod.F90 similarity index 100% rename from driver_cpl/unit_test/utils/create_mapper_mod.F90 rename to src/drivers/mct/unit_test/utils/create_mapper_mod.F90 diff --git a/driver_cpl/unit_test/utils/mct_wrapper_mod.F90 b/src/drivers/mct/unit_test/utils/mct_wrapper_mod.F90 similarity index 100% rename from driver_cpl/unit_test/utils/mct_wrapper_mod.F90 rename to src/drivers/mct/unit_test/utils/mct_wrapper_mod.F90 diff --git a/driver_cpl/unit_test/utils/simple_map_mod.F90 b/src/drivers/mct/unit_test/utils/simple_map_mod.F90 similarity index 98% rename from driver_cpl/unit_test/utils/simple_map_mod.F90 rename to src/drivers/mct/unit_test/utils/simple_map_mod.F90 index 8738a86ef6f..485dca44481 100644 --- a/driver_cpl/unit_test/utils/simple_map_mod.F90 +++ b/src/drivers/mct/unit_test/utils/simple_map_mod.F90 @@ -66,8 +66,8 @@ function constructor(source_indices, dest_indices, overlap_weights) result(this) !----------------------------------------------------------------------- n_overlaps = size(overlap_weights) - call shr_assert(size(source_indices) == n_overlaps, errMsg(__FILE__, __LINE__)) - call shr_assert(size(dest_indices) == n_overlaps, errMsg(__FILE__, __LINE__)) + call shr_assert(size(source_indices) == n_overlaps, file=__FILE__, line=__LINE__) + call shr_assert(size(dest_indices) == n_overlaps, file=__FILE__, line=__LINE__) this%n_overlaps = n_overlaps this%source_indices = source_indices diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/CMakeLists.txt b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/CMakeLists.txt rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/README b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/README similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/README rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/README diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/gradient_example.txt b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/gradient_example.txt similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/gradient_example.txt rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/gradient_example.txt diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/plot_gradient b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/plot_gradient similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/plot_gradient rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/plot_gradient diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf diff --git a/externals/CMake/.gitignore b/src/externals/CMake/.gitignore similarity index 100% rename from externals/CMake/.gitignore rename to src/externals/CMake/.gitignore diff --git a/externals/CMake/CESM_utils.cmake b/src/externals/CMake/CESM_utils.cmake similarity index 100% rename from externals/CMake/CESM_utils.cmake rename to src/externals/CMake/CESM_utils.cmake diff --git a/externals/CMake/CIME_utils.cmake b/src/externals/CMake/CIME_utils.cmake similarity index 100% rename from externals/CMake/CIME_utils.cmake rename to src/externals/CMake/CIME_utils.cmake diff --git a/externals/CMake/ChangeLog b/src/externals/CMake/ChangeLog similarity index 100% rename from externals/CMake/ChangeLog rename to src/externals/CMake/ChangeLog diff --git a/externals/CMake/Compilers.cmake b/src/externals/CMake/Compilers.cmake similarity index 100% rename from externals/CMake/Compilers.cmake rename to src/externals/CMake/Compilers.cmake diff --git a/externals/CMake/FindpFUnit.cmake b/src/externals/CMake/FindpFUnit.cmake similarity index 100% rename from externals/CMake/FindpFUnit.cmake rename to src/externals/CMake/FindpFUnit.cmake diff --git a/externals/CMake/LICENSE b/src/externals/CMake/LICENSE similarity index 100% rename from externals/CMake/LICENSE rename to src/externals/CMake/LICENSE diff --git a/externals/CMake/README.md b/src/externals/CMake/README.md similarity index 100% rename from externals/CMake/README.md rename to src/externals/CMake/README.md diff --git a/externals/CMake/Sourcelist_utils.cmake b/src/externals/CMake/Sourcelist_utils.cmake similarity index 100% rename from externals/CMake/Sourcelist_utils.cmake rename to src/externals/CMake/Sourcelist_utils.cmake diff --git a/externals/CMake/TryCSizeOf.f90 b/src/externals/CMake/TryCSizeOf.f90 similarity index 100% rename from externals/CMake/TryCSizeOf.f90 rename to src/externals/CMake/TryCSizeOf.f90 diff --git a/externals/CMake/TryMPIIO.f90 b/src/externals/CMake/TryMPIIO.f90 similarity index 100% rename from externals/CMake/TryMPIIO.f90 rename to src/externals/CMake/TryMPIIO.f90 diff --git a/externals/CMake/TryMPIMod.f90 b/src/externals/CMake/TryMPIMod.f90 similarity index 100% rename from externals/CMake/TryMPIMod.f90 rename to src/externals/CMake/TryMPIMod.f90 diff --git a/externals/CMake/TryMPISERIAL.f90 b/src/externals/CMake/TryMPISERIAL.f90 similarity index 100% rename from externals/CMake/TryMPISERIAL.f90 rename to src/externals/CMake/TryMPISERIAL.f90 diff --git a/externals/CMake/TryPnetcdf_inc.f90 b/src/externals/CMake/TryPnetcdf_inc.f90 similarity index 100% rename from externals/CMake/TryPnetcdf_inc.f90 rename to src/externals/CMake/TryPnetcdf_inc.f90 diff --git a/externals/CMake/TryPnetcdf_mod.f90 b/src/externals/CMake/TryPnetcdf_mod.f90 similarity index 100% rename from externals/CMake/TryPnetcdf_mod.f90 rename to src/externals/CMake/TryPnetcdf_mod.f90 diff --git a/externals/CMake/genf90_utils.cmake b/src/externals/CMake/genf90_utils.cmake similarity index 100% rename from externals/CMake/genf90_utils.cmake rename to src/externals/CMake/genf90_utils.cmake diff --git a/externals/CMake/mpiexec.cmake b/src/externals/CMake/mpiexec.cmake similarity index 100% rename from externals/CMake/mpiexec.cmake rename to src/externals/CMake/mpiexec.cmake diff --git a/externals/CMake/pFUnit_utils.cmake b/src/externals/CMake/pFUnit_utils.cmake similarity index 100% rename from externals/CMake/pFUnit_utils.cmake rename to src/externals/CMake/pFUnit_utils.cmake diff --git a/externals/genf90/ChangeLog b/src/externals/genf90/ChangeLog similarity index 100% rename from externals/genf90/ChangeLog rename to src/externals/genf90/ChangeLog diff --git a/externals/genf90/genf90.pl b/src/externals/genf90/genf90.pl similarity index 100% rename from externals/genf90/genf90.pl rename to src/externals/genf90/genf90.pl diff --git a/externals/mct/.gitignore b/src/externals/mct/.gitignore similarity index 100% rename from externals/mct/.gitignore rename to src/externals/mct/.gitignore diff --git a/externals/mct/COPYRIGHT b/src/externals/mct/COPYRIGHT similarity index 100% rename from externals/mct/COPYRIGHT rename to src/externals/mct/COPYRIGHT diff --git a/externals/mct/Makefile b/src/externals/mct/Makefile similarity index 100% rename from externals/mct/Makefile rename to src/externals/mct/Makefile diff --git a/externals/mct/Makefile.conf.in b/src/externals/mct/Makefile.conf.in similarity index 100% rename from externals/mct/Makefile.conf.in rename to src/externals/mct/Makefile.conf.in diff --git a/externals/mct/README b/src/externals/mct/README similarity index 100% rename from externals/mct/README rename to src/externals/mct/README diff --git a/externals/mct/aclocal.m4 b/src/externals/mct/aclocal.m4 similarity index 100% rename from externals/mct/aclocal.m4 rename to src/externals/mct/aclocal.m4 diff --git a/externals/mct/benchmarks/.gitignore b/src/externals/mct/benchmarks/.gitignore similarity index 100% rename from externals/mct/benchmarks/.gitignore rename to src/externals/mct/benchmarks/.gitignore diff --git a/externals/mct/benchmarks/Makefile b/src/externals/mct/benchmarks/Makefile similarity index 100% rename from externals/mct/benchmarks/Makefile rename to src/externals/mct/benchmarks/Makefile diff --git a/externals/mct/benchmarks/RouterTestDis.F90 b/src/externals/mct/benchmarks/RouterTestDis.F90 similarity index 100% rename from externals/mct/benchmarks/RouterTestDis.F90 rename to src/externals/mct/benchmarks/RouterTestDis.F90 diff --git a/externals/mct/benchmarks/RouterTestOvr.F90 b/src/externals/mct/benchmarks/RouterTestOvr.F90 similarity index 100% rename from externals/mct/benchmarks/RouterTestOvr.F90 rename to src/externals/mct/benchmarks/RouterTestOvr.F90 diff --git a/externals/mct/benchmarks/T42.8pC b/src/externals/mct/benchmarks/T42.8pC similarity index 100% rename from externals/mct/benchmarks/T42.8pC rename to src/externals/mct/benchmarks/T42.8pC diff --git a/externals/mct/benchmarks/T42.8pR b/src/externals/mct/benchmarks/T42.8pR similarity index 100% rename from externals/mct/benchmarks/T42.8pR rename to src/externals/mct/benchmarks/T42.8pR diff --git a/externals/mct/benchmarks/gx1.8pC b/src/externals/mct/benchmarks/gx1.8pC similarity index 100% rename from externals/mct/benchmarks/gx1.8pC rename to src/externals/mct/benchmarks/gx1.8pC diff --git a/externals/mct/benchmarks/gx1.8pR b/src/externals/mct/benchmarks/gx1.8pR similarity index 100% rename from externals/mct/benchmarks/gx1.8pR rename to src/externals/mct/benchmarks/gx1.8pR diff --git a/externals/mct/benchmarks/importBench.F90 b/src/externals/mct/benchmarks/importBench.F90 similarity index 100% rename from externals/mct/benchmarks/importBench.F90 rename to src/externals/mct/benchmarks/importBench.F90 diff --git a/externals/mct/config.h.in b/src/externals/mct/config.h.in similarity index 100% rename from externals/mct/config.h.in rename to src/externals/mct/config.h.in diff --git a/externals/mct/configure b/src/externals/mct/configure similarity index 100% rename from externals/mct/configure rename to src/externals/mct/configure diff --git a/externals/mct/configure.ac b/src/externals/mct/configure.ac similarity index 100% rename from externals/mct/configure.ac rename to src/externals/mct/configure.ac diff --git a/externals/mct/doc/.gitignore b/src/externals/mct/doc/.gitignore similarity index 100% rename from externals/mct/doc/.gitignore rename to src/externals/mct/doc/.gitignore diff --git a/externals/mct/doc/Makefile b/src/externals/mct/doc/Makefile similarity index 100% rename from externals/mct/doc/Makefile rename to src/externals/mct/doc/Makefile diff --git a/externals/mct/doc/README b/src/externals/mct/doc/README similarity index 100% rename from externals/mct/doc/README rename to src/externals/mct/doc/README diff --git a/externals/mct/doc/coupler.bib b/src/externals/mct/doc/coupler.bib similarity index 100% rename from externals/mct/doc/coupler.bib rename to src/externals/mct/doc/coupler.bib diff --git a/externals/mct/doc/mct_APIs.tex b/src/externals/mct/doc/mct_APIs.tex similarity index 100% rename from externals/mct/doc/mct_APIs.tex rename to src/externals/mct/doc/mct_APIs.tex diff --git a/externals/mct/doc/texsrc/.gitignore b/src/externals/mct/doc/texsrc/.gitignore similarity index 100% rename from externals/mct/doc/texsrc/.gitignore rename to src/externals/mct/doc/texsrc/.gitignore diff --git a/externals/mct/doc/texsrc/Makefile b/src/externals/mct/doc/texsrc/Makefile similarity index 100% rename from externals/mct/doc/texsrc/Makefile rename to src/externals/mct/doc/texsrc/Makefile diff --git a/externals/mct/doc/texsrc/SRCS_tex.mk b/src/externals/mct/doc/texsrc/SRCS_tex.mk similarity index 100% rename from externals/mct/doc/texsrc/SRCS_tex.mk rename to src/externals/mct/doc/texsrc/SRCS_tex.mk diff --git a/externals/mct/examples/Makefile b/src/externals/mct/examples/Makefile similarity index 100% rename from externals/mct/examples/Makefile rename to src/externals/mct/examples/Makefile diff --git a/externals/mct/examples/README b/src/externals/mct/examples/README similarity index 100% rename from externals/mct/examples/README rename to src/externals/mct/examples/README diff --git a/externals/mct/examples/climate_concur1/.gitignore b/src/externals/mct/examples/climate_concur1/.gitignore similarity index 100% rename from externals/mct/examples/climate_concur1/.gitignore rename to src/externals/mct/examples/climate_concur1/.gitignore diff --git a/externals/mct/examples/climate_concur1/Makefile b/src/externals/mct/examples/climate_concur1/Makefile similarity index 100% rename from externals/mct/examples/climate_concur1/Makefile rename to src/externals/mct/examples/climate_concur1/Makefile diff --git a/externals/mct/examples/climate_concur1/README b/src/externals/mct/examples/climate_concur1/README similarity index 100% rename from externals/mct/examples/climate_concur1/README rename to src/externals/mct/examples/climate_concur1/README diff --git a/externals/mct/examples/climate_concur1/coupler.F90 b/src/externals/mct/examples/climate_concur1/coupler.F90 similarity index 100% rename from externals/mct/examples/climate_concur1/coupler.F90 rename to src/externals/mct/examples/climate_concur1/coupler.F90 diff --git a/externals/mct/examples/climate_concur1/master.F90 b/src/externals/mct/examples/climate_concur1/master.F90 similarity index 100% rename from externals/mct/examples/climate_concur1/master.F90 rename to src/externals/mct/examples/climate_concur1/master.F90 diff --git a/externals/mct/examples/climate_concur1/model.F90 b/src/externals/mct/examples/climate_concur1/model.F90 similarity index 100% rename from externals/mct/examples/climate_concur1/model.F90 rename to src/externals/mct/examples/climate_concur1/model.F90 diff --git a/externals/mct/examples/climate_sequen1/.gitignore b/src/externals/mct/examples/climate_sequen1/.gitignore similarity index 100% rename from externals/mct/examples/climate_sequen1/.gitignore rename to src/externals/mct/examples/climate_sequen1/.gitignore diff --git a/externals/mct/examples/climate_sequen1/Makefile b/src/externals/mct/examples/climate_sequen1/Makefile similarity index 100% rename from externals/mct/examples/climate_sequen1/Makefile rename to src/externals/mct/examples/climate_sequen1/Makefile diff --git a/externals/mct/examples/climate_sequen1/README b/src/externals/mct/examples/climate_sequen1/README similarity index 100% rename from externals/mct/examples/climate_sequen1/README rename to src/externals/mct/examples/climate_sequen1/README diff --git a/externals/mct/examples/climate_sequen1/TS1.dat b/src/externals/mct/examples/climate_sequen1/TS1.dat similarity index 100% rename from externals/mct/examples/climate_sequen1/TS1.dat rename to src/externals/mct/examples/climate_sequen1/TS1.dat diff --git a/externals/mct/examples/climate_sequen1/coupler.F90 b/src/externals/mct/examples/climate_sequen1/coupler.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/coupler.F90 rename to src/externals/mct/examples/climate_sequen1/coupler.F90 diff --git a/externals/mct/examples/climate_sequen1/dst.rc b/src/externals/mct/examples/climate_sequen1/dst.rc similarity index 100% rename from externals/mct/examples/climate_sequen1/dst.rc rename to src/externals/mct/examples/climate_sequen1/dst.rc diff --git a/externals/mct/examples/climate_sequen1/dstmodel.F90 b/src/externals/mct/examples/climate_sequen1/dstmodel.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/dstmodel.F90 rename to src/externals/mct/examples/climate_sequen1/dstmodel.F90 diff --git a/externals/mct/examples/climate_sequen1/master.F90 b/src/externals/mct/examples/climate_sequen1/master.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/master.F90 rename to src/externals/mct/examples/climate_sequen1/master.F90 diff --git a/externals/mct/examples/climate_sequen1/mutils.F90 b/src/externals/mct/examples/climate_sequen1/mutils.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/mutils.F90 rename to src/externals/mct/examples/climate_sequen1/mutils.F90 diff --git a/externals/mct/examples/climate_sequen1/src.rc b/src/externals/mct/examples/climate_sequen1/src.rc similarity index 100% rename from externals/mct/examples/climate_sequen1/src.rc rename to src/externals/mct/examples/climate_sequen1/src.rc diff --git a/externals/mct/examples/climate_sequen1/srcmodel.F90 b/src/externals/mct/examples/climate_sequen1/srcmodel.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/srcmodel.F90 rename to src/externals/mct/examples/climate_sequen1/srcmodel.F90 diff --git a/externals/mct/examples/simple/.gitignore b/src/externals/mct/examples/simple/.gitignore similarity index 100% rename from externals/mct/examples/simple/.gitignore rename to src/externals/mct/examples/simple/.gitignore diff --git a/externals/mct/examples/simple/Makefile b/src/externals/mct/examples/simple/Makefile similarity index 100% rename from externals/mct/examples/simple/Makefile rename to src/externals/mct/examples/simple/Makefile diff --git a/externals/mct/examples/simple/README b/src/externals/mct/examples/simple/README similarity index 100% rename from externals/mct/examples/simple/README rename to src/externals/mct/examples/simple/README diff --git a/externals/mct/examples/simple/script.babyblue b/src/externals/mct/examples/simple/script.babyblue similarity index 100% rename from externals/mct/examples/simple/script.babyblue rename to src/externals/mct/examples/simple/script.babyblue diff --git a/externals/mct/examples/simple/twocmp.con.F90 b/src/externals/mct/examples/simple/twocmp.con.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.con.F90 rename to src/externals/mct/examples/simple/twocmp.con.F90 diff --git a/externals/mct/examples/simple/twocmp.seq.F90 b/src/externals/mct/examples/simple/twocmp.seq.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.seq.F90 rename to src/externals/mct/examples/simple/twocmp.seq.F90 diff --git a/externals/mct/examples/simple/twocmp.seqNB.F90 b/src/externals/mct/examples/simple/twocmp.seqNB.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.seqNB.F90 rename to src/externals/mct/examples/simple/twocmp.seqNB.F90 diff --git a/externals/mct/examples/simple/twocmp.seqUnvn.F90 b/src/externals/mct/examples/simple/twocmp.seqUnvn.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.seqUnvn.F90 rename to src/externals/mct/examples/simple/twocmp.seqUnvn.F90 diff --git a/externals/mct/install-sh b/src/externals/mct/install-sh similarity index 100% rename from externals/mct/install-sh rename to src/externals/mct/install-sh diff --git a/externals/mct/m4/README b/src/externals/mct/m4/README similarity index 100% rename from externals/mct/m4/README rename to src/externals/mct/m4/README diff --git a/externals/mct/m4/acx_mpi.m4 b/src/externals/mct/m4/acx_mpi.m4 similarity index 100% rename from externals/mct/m4/acx_mpi.m4 rename to src/externals/mct/m4/acx_mpi.m4 diff --git a/externals/mct/m4/ax_fc_version.m4 b/src/externals/mct/m4/ax_fc_version.m4 similarity index 100% rename from externals/mct/m4/ax_fc_version.m4 rename to src/externals/mct/m4/ax_fc_version.m4 diff --git a/externals/mct/m4/fortran.m4 b/src/externals/mct/m4/fortran.m4 similarity index 100% rename from externals/mct/m4/fortran.m4 rename to src/externals/mct/m4/fortran.m4 diff --git a/externals/mct/mct/Makefile b/src/externals/mct/mct/Makefile similarity index 100% rename from externals/mct/mct/Makefile rename to src/externals/mct/mct/Makefile diff --git a/externals/mct/mct/README b/src/externals/mct/mct/README similarity index 100% rename from externals/mct/mct/README rename to src/externals/mct/mct/README diff --git a/externals/mct/mct/m_Accumulator.F90 b/src/externals/mct/mct/m_Accumulator.F90 similarity index 100% rename from externals/mct/mct/m_Accumulator.F90 rename to src/externals/mct/mct/m_Accumulator.F90 diff --git a/externals/mct/mct/m_AccumulatorComms.F90 b/src/externals/mct/mct/m_AccumulatorComms.F90 similarity index 100% rename from externals/mct/mct/m_AccumulatorComms.F90 rename to src/externals/mct/mct/m_AccumulatorComms.F90 diff --git a/externals/mct/mct/m_AttrVect.F90 b/src/externals/mct/mct/m_AttrVect.F90 similarity index 100% rename from externals/mct/mct/m_AttrVect.F90 rename to src/externals/mct/mct/m_AttrVect.F90 diff --git a/externals/mct/mct/m_AttrVectComms.F90 b/src/externals/mct/mct/m_AttrVectComms.F90 similarity index 100% rename from externals/mct/mct/m_AttrVectComms.F90 rename to src/externals/mct/mct/m_AttrVectComms.F90 diff --git a/externals/mct/mct/m_AttrVectReduce.F90 b/src/externals/mct/mct/m_AttrVectReduce.F90 similarity index 100% rename from externals/mct/mct/m_AttrVectReduce.F90 rename to src/externals/mct/mct/m_AttrVectReduce.F90 diff --git a/externals/mct/mct/m_ConvertMaps.F90 b/src/externals/mct/mct/m_ConvertMaps.F90 similarity index 100% rename from externals/mct/mct/m_ConvertMaps.F90 rename to src/externals/mct/mct/m_ConvertMaps.F90 diff --git a/externals/mct/mct/m_ExchangeMaps.F90 b/src/externals/mct/mct/m_ExchangeMaps.F90 similarity index 100% rename from externals/mct/mct/m_ExchangeMaps.F90 rename to src/externals/mct/mct/m_ExchangeMaps.F90 diff --git a/externals/mct/mct/m_GeneralGrid.F90 b/src/externals/mct/mct/m_GeneralGrid.F90 similarity index 100% rename from externals/mct/mct/m_GeneralGrid.F90 rename to src/externals/mct/mct/m_GeneralGrid.F90 diff --git a/externals/mct/mct/m_GeneralGridComms.F90 b/src/externals/mct/mct/m_GeneralGridComms.F90 similarity index 100% rename from externals/mct/mct/m_GeneralGridComms.F90 rename to src/externals/mct/mct/m_GeneralGridComms.F90 diff --git a/externals/mct/mct/m_GlobalMap.F90 b/src/externals/mct/mct/m_GlobalMap.F90 similarity index 100% rename from externals/mct/mct/m_GlobalMap.F90 rename to src/externals/mct/mct/m_GlobalMap.F90 diff --git a/externals/mct/mct/m_GlobalSegMap.F90 b/src/externals/mct/mct/m_GlobalSegMap.F90 similarity index 100% rename from externals/mct/mct/m_GlobalSegMap.F90 rename to src/externals/mct/mct/m_GlobalSegMap.F90 diff --git a/externals/mct/mct/m_GlobalSegMapComms.F90 b/src/externals/mct/mct/m_GlobalSegMapComms.F90 similarity index 100% rename from externals/mct/mct/m_GlobalSegMapComms.F90 rename to src/externals/mct/mct/m_GlobalSegMapComms.F90 diff --git a/externals/mct/mct/m_GlobalToLocal.F90 b/src/externals/mct/mct/m_GlobalToLocal.F90 similarity index 100% rename from externals/mct/mct/m_GlobalToLocal.F90 rename to src/externals/mct/mct/m_GlobalToLocal.F90 diff --git a/externals/mct/mct/m_MCTWorld.F90 b/src/externals/mct/mct/m_MCTWorld.F90 similarity index 100% rename from externals/mct/mct/m_MCTWorld.F90 rename to src/externals/mct/mct/m_MCTWorld.F90 diff --git a/externals/mct/mct/m_MatAttrVectMul.F90 b/src/externals/mct/mct/m_MatAttrVectMul.F90 similarity index 100% rename from externals/mct/mct/m_MatAttrVectMul.F90 rename to src/externals/mct/mct/m_MatAttrVectMul.F90 diff --git a/externals/mct/mct/m_Merge.F90 b/src/externals/mct/mct/m_Merge.F90 similarity index 100% rename from externals/mct/mct/m_Merge.F90 rename to src/externals/mct/mct/m_Merge.F90 diff --git a/externals/mct/mct/m_Navigator.F90 b/src/externals/mct/mct/m_Navigator.F90 similarity index 100% rename from externals/mct/mct/m_Navigator.F90 rename to src/externals/mct/mct/m_Navigator.F90 diff --git a/externals/mct/mct/m_Rearranger.F90 b/src/externals/mct/mct/m_Rearranger.F90 similarity index 100% rename from externals/mct/mct/m_Rearranger.F90 rename to src/externals/mct/mct/m_Rearranger.F90 diff --git a/externals/mct/mct/m_Router.F90 b/src/externals/mct/mct/m_Router.F90 similarity index 100% rename from externals/mct/mct/m_Router.F90 rename to src/externals/mct/mct/m_Router.F90 diff --git a/externals/mct/mct/m_SparseMatrix.F90 b/src/externals/mct/mct/m_SparseMatrix.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrix.F90 rename to src/externals/mct/mct/m_SparseMatrix.F90 diff --git a/externals/mct/mct/m_SparseMatrixComms.F90 b/src/externals/mct/mct/m_SparseMatrixComms.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixComms.F90 rename to src/externals/mct/mct/m_SparseMatrixComms.F90 diff --git a/externals/mct/mct/m_SparseMatrixDecomp.F90 b/src/externals/mct/mct/m_SparseMatrixDecomp.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixDecomp.F90 rename to src/externals/mct/mct/m_SparseMatrixDecomp.F90 diff --git a/externals/mct/mct/m_SparseMatrixPlus.F90 b/src/externals/mct/mct/m_SparseMatrixPlus.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixPlus.F90 rename to src/externals/mct/mct/m_SparseMatrixPlus.F90 diff --git a/externals/mct/mct/m_SparseMatrixToMaps.F90 b/src/externals/mct/mct/m_SparseMatrixToMaps.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixToMaps.F90 rename to src/externals/mct/mct/m_SparseMatrixToMaps.F90 diff --git a/externals/mct/mct/m_SpatialIntegral.F90 b/src/externals/mct/mct/m_SpatialIntegral.F90 similarity index 100% rename from externals/mct/mct/m_SpatialIntegral.F90 rename to src/externals/mct/mct/m_SpatialIntegral.F90 diff --git a/externals/mct/mct/m_SpatialIntegralV.F90 b/src/externals/mct/mct/m_SpatialIntegralV.F90 similarity index 100% rename from externals/mct/mct/m_SpatialIntegralV.F90 rename to src/externals/mct/mct/m_SpatialIntegralV.F90 diff --git a/externals/mct/mct/m_Transfer.F90 b/src/externals/mct/mct/m_Transfer.F90 similarity index 100% rename from externals/mct/mct/m_Transfer.F90 rename to src/externals/mct/mct/m_Transfer.F90 diff --git a/externals/mct/mkinstalldirs b/src/externals/mct/mkinstalldirs similarity index 100% rename from externals/mct/mkinstalldirs rename to src/externals/mct/mkinstalldirs diff --git a/externals/mct/mpeu/Makefile b/src/externals/mct/mpeu/Makefile similarity index 100% rename from externals/mct/mpeu/Makefile rename to src/externals/mct/mpeu/Makefile diff --git a/externals/mct/mpeu/README b/src/externals/mct/mpeu/README similarity index 100% rename from externals/mct/mpeu/README rename to src/externals/mct/mpeu/README diff --git a/externals/mct/mpeu/assertmpeu.H b/src/externals/mct/mpeu/assertmpeu.H similarity index 100% rename from externals/mct/mpeu/assertmpeu.H rename to src/externals/mct/mpeu/assertmpeu.H diff --git a/externals/mct/mpeu/get_zeits.c b/src/externals/mct/mpeu/get_zeits.c similarity index 100% rename from externals/mct/mpeu/get_zeits.c rename to src/externals/mct/mpeu/get_zeits.c diff --git a/externals/mct/mpeu/m_FcComms.F90 b/src/externals/mct/mpeu/m_FcComms.F90 similarity index 100% rename from externals/mct/mpeu/m_FcComms.F90 rename to src/externals/mct/mpeu/m_FcComms.F90 diff --git a/externals/mct/mpeu/m_FileResolv.F90 b/src/externals/mct/mpeu/m_FileResolv.F90 similarity index 100% rename from externals/mct/mpeu/m_FileResolv.F90 rename to src/externals/mct/mpeu/m_FileResolv.F90 diff --git a/externals/mct/mpeu/m_Filename.F90 b/src/externals/mct/mpeu/m_Filename.F90 similarity index 100% rename from externals/mct/mpeu/m_Filename.F90 rename to src/externals/mct/mpeu/m_Filename.F90 diff --git a/externals/mct/mpeu/m_IndexBin_char.F90 b/src/externals/mct/mpeu/m_IndexBin_char.F90 similarity index 100% rename from externals/mct/mpeu/m_IndexBin_char.F90 rename to src/externals/mct/mpeu/m_IndexBin_char.F90 diff --git a/externals/mct/mpeu/m_IndexBin_integer.F90 b/src/externals/mct/mpeu/m_IndexBin_integer.F90 similarity index 100% rename from externals/mct/mpeu/m_IndexBin_integer.F90 rename to src/externals/mct/mpeu/m_IndexBin_integer.F90 diff --git a/externals/mct/mpeu/m_IndexBin_logical.F90 b/src/externals/mct/mpeu/m_IndexBin_logical.F90 similarity index 100% rename from externals/mct/mpeu/m_IndexBin_logical.F90 rename to src/externals/mct/mpeu/m_IndexBin_logical.F90 diff --git a/externals/mct/mpeu/m_List.F90 b/src/externals/mct/mpeu/m_List.F90 similarity index 100% rename from externals/mct/mpeu/m_List.F90 rename to src/externals/mct/mpeu/m_List.F90 diff --git a/externals/mct/mpeu/m_MergeSorts.F90 b/src/externals/mct/mpeu/m_MergeSorts.F90 similarity index 100% rename from externals/mct/mpeu/m_MergeSorts.F90 rename to src/externals/mct/mpeu/m_MergeSorts.F90 diff --git a/externals/mct/mpeu/m_Permuter.F90 b/src/externals/mct/mpeu/m_Permuter.F90 similarity index 100% rename from externals/mct/mpeu/m_Permuter.F90 rename to src/externals/mct/mpeu/m_Permuter.F90 diff --git a/externals/mct/mpeu/m_SortingTools.F90 b/src/externals/mct/mpeu/m_SortingTools.F90 similarity index 100% rename from externals/mct/mpeu/m_SortingTools.F90 rename to src/externals/mct/mpeu/m_SortingTools.F90 diff --git a/externals/mct/mpeu/m_StrTemplate.F90 b/src/externals/mct/mpeu/m_StrTemplate.F90 similarity index 100% rename from externals/mct/mpeu/m_StrTemplate.F90 rename to src/externals/mct/mpeu/m_StrTemplate.F90 diff --git a/externals/mct/mpeu/m_String.F90 b/src/externals/mct/mpeu/m_String.F90 similarity index 100% rename from externals/mct/mpeu/m_String.F90 rename to src/externals/mct/mpeu/m_String.F90 diff --git a/externals/mct/mpeu/m_StringLinkedList.F90 b/src/externals/mct/mpeu/m_StringLinkedList.F90 similarity index 100% rename from externals/mct/mpeu/m_StringLinkedList.F90 rename to src/externals/mct/mpeu/m_StringLinkedList.F90 diff --git a/externals/mct/mpeu/m_TraceBack.F90 b/src/externals/mct/mpeu/m_TraceBack.F90 similarity index 100% rename from externals/mct/mpeu/m_TraceBack.F90 rename to src/externals/mct/mpeu/m_TraceBack.F90 diff --git a/externals/mct/mpeu/m_chars.F90 b/src/externals/mct/mpeu/m_chars.F90 similarity index 100% rename from externals/mct/mpeu/m_chars.F90 rename to src/externals/mct/mpeu/m_chars.F90 diff --git a/externals/mct/mpeu/m_die.F90 b/src/externals/mct/mpeu/m_die.F90 similarity index 100% rename from externals/mct/mpeu/m_die.F90 rename to src/externals/mct/mpeu/m_die.F90 diff --git a/externals/mct/mpeu/m_dropdead.F90 b/src/externals/mct/mpeu/m_dropdead.F90 similarity index 100% rename from externals/mct/mpeu/m_dropdead.F90 rename to src/externals/mct/mpeu/m_dropdead.F90 diff --git a/externals/mct/mpeu/m_flow.F90 b/src/externals/mct/mpeu/m_flow.F90 similarity index 100% rename from externals/mct/mpeu/m_flow.F90 rename to src/externals/mct/mpeu/m_flow.F90 diff --git a/externals/mct/mpeu/m_inpak90.F90 b/src/externals/mct/mpeu/m_inpak90.F90 similarity index 100% rename from externals/mct/mpeu/m_inpak90.F90 rename to src/externals/mct/mpeu/m_inpak90.F90 diff --git a/externals/mct/mpeu/m_ioutil.F90 b/src/externals/mct/mpeu/m_ioutil.F90 similarity index 100% rename from externals/mct/mpeu/m_ioutil.F90 rename to src/externals/mct/mpeu/m_ioutil.F90 diff --git a/externals/mct/mpeu/m_mall.F90 b/src/externals/mct/mpeu/m_mall.F90 similarity index 100% rename from externals/mct/mpeu/m_mall.F90 rename to src/externals/mct/mpeu/m_mall.F90 diff --git a/externals/mct/mpeu/m_mpif.F90 b/src/externals/mct/mpeu/m_mpif.F90 similarity index 100% rename from externals/mct/mpeu/m_mpif.F90 rename to src/externals/mct/mpeu/m_mpif.F90 diff --git a/externals/mct/mpeu/m_mpif90.F90 b/src/externals/mct/mpeu/m_mpif90.F90 similarity index 100% rename from externals/mct/mpeu/m_mpif90.F90 rename to src/externals/mct/mpeu/m_mpif90.F90 diff --git a/externals/mct/mpeu/m_mpout.F90 b/src/externals/mct/mpeu/m_mpout.F90 similarity index 100% rename from externals/mct/mpeu/m_mpout.F90 rename to src/externals/mct/mpeu/m_mpout.F90 diff --git a/externals/mct/mpeu/m_rankMerge.F90 b/src/externals/mct/mpeu/m_rankMerge.F90 similarity index 100% rename from externals/mct/mpeu/m_rankMerge.F90 rename to src/externals/mct/mpeu/m_rankMerge.F90 diff --git a/externals/mct/mpeu/m_realkinds.F90 b/src/externals/mct/mpeu/m_realkinds.F90 similarity index 100% rename from externals/mct/mpeu/m_realkinds.F90 rename to src/externals/mct/mpeu/m_realkinds.F90 diff --git a/externals/mct/mpeu/m_stdio.F90 b/src/externals/mct/mpeu/m_stdio.F90 similarity index 100% rename from externals/mct/mpeu/m_stdio.F90 rename to src/externals/mct/mpeu/m_stdio.F90 diff --git a/externals/mct/mpeu/m_zeit.F90 b/src/externals/mct/mpeu/m_zeit.F90 similarity index 100% rename from externals/mct/mpeu/m_zeit.F90 rename to src/externals/mct/mpeu/m_zeit.F90 diff --git a/externals/mct/mpi-serial/.gitignore b/src/externals/mct/mpi-serial/.gitignore similarity index 100% rename from externals/mct/mpi-serial/.gitignore rename to src/externals/mct/mpi-serial/.gitignore diff --git a/externals/mct/mpi-serial/Makefile b/src/externals/mct/mpi-serial/Makefile similarity index 100% rename from externals/mct/mpi-serial/Makefile rename to src/externals/mct/mpi-serial/Makefile diff --git a/externals/mct/mpi-serial/Makefile.conf.in b/src/externals/mct/mpi-serial/Makefile.conf.in similarity index 100% rename from externals/mct/mpi-serial/Makefile.conf.in rename to src/externals/mct/mpi-serial/Makefile.conf.in diff --git a/externals/mct/mpi-serial/NOTES b/src/externals/mct/mpi-serial/NOTES similarity index 100% rename from externals/mct/mpi-serial/NOTES rename to src/externals/mct/mpi-serial/NOTES diff --git a/externals/mct/mpi-serial/README b/src/externals/mct/mpi-serial/README similarity index 100% rename from externals/mct/mpi-serial/README rename to src/externals/mct/mpi-serial/README diff --git a/externals/mct/mpi-serial/aclocal.m4 b/src/externals/mct/mpi-serial/aclocal.m4 similarity index 100% rename from externals/mct/mpi-serial/aclocal.m4 rename to src/externals/mct/mpi-serial/aclocal.m4 diff --git a/externals/mct/mpi-serial/cart.c b/src/externals/mct/mpi-serial/cart.c similarity index 100% rename from externals/mct/mpi-serial/cart.c rename to src/externals/mct/mpi-serial/cart.c diff --git a/externals/mct/mpi-serial/collective.c b/src/externals/mct/mpi-serial/collective.c similarity index 100% rename from externals/mct/mpi-serial/collective.c rename to src/externals/mct/mpi-serial/collective.c diff --git a/externals/mct/mpi-serial/comm.c b/src/externals/mct/mpi-serial/comm.c similarity index 100% rename from externals/mct/mpi-serial/comm.c rename to src/externals/mct/mpi-serial/comm.c diff --git a/externals/mct/mpi-serial/config.h.in b/src/externals/mct/mpi-serial/config.h.in similarity index 100% rename from externals/mct/mpi-serial/config.h.in rename to src/externals/mct/mpi-serial/config.h.in diff --git a/externals/mct/mpi-serial/configure b/src/externals/mct/mpi-serial/configure similarity index 100% rename from externals/mct/mpi-serial/configure rename to src/externals/mct/mpi-serial/configure diff --git a/externals/mct/mpi-serial/configure.in b/src/externals/mct/mpi-serial/configure.in similarity index 100% rename from externals/mct/mpi-serial/configure.in rename to src/externals/mct/mpi-serial/configure.in diff --git a/externals/mct/mpi-serial/copy.c b/src/externals/mct/mpi-serial/copy.c similarity index 100% rename from externals/mct/mpi-serial/copy.c rename to src/externals/mct/mpi-serial/copy.c diff --git a/externals/mct/mpi-serial/fort.F90 b/src/externals/mct/mpi-serial/fort.F90 similarity index 100% rename from externals/mct/mpi-serial/fort.F90 rename to src/externals/mct/mpi-serial/fort.F90 diff --git a/externals/mct/mpi-serial/getcount.c b/src/externals/mct/mpi-serial/getcount.c similarity index 100% rename from externals/mct/mpi-serial/getcount.c rename to src/externals/mct/mpi-serial/getcount.c diff --git a/externals/mct/mpi-serial/group.c b/src/externals/mct/mpi-serial/group.c similarity index 100% rename from externals/mct/mpi-serial/group.c rename to src/externals/mct/mpi-serial/group.c diff --git a/externals/mct/mpi-serial/handles.c b/src/externals/mct/mpi-serial/handles.c similarity index 100% rename from externals/mct/mpi-serial/handles.c rename to src/externals/mct/mpi-serial/handles.c diff --git a/externals/mct/mpi-serial/info.c b/src/externals/mct/mpi-serial/info.c similarity index 100% rename from externals/mct/mpi-serial/info.c rename to src/externals/mct/mpi-serial/info.c diff --git a/externals/mct/mpi-serial/list.c b/src/externals/mct/mpi-serial/list.c similarity index 100% rename from externals/mct/mpi-serial/list.c rename to src/externals/mct/mpi-serial/list.c diff --git a/externals/mct/mpi-serial/list.h b/src/externals/mct/mpi-serial/list.h similarity index 100% rename from externals/mct/mpi-serial/list.h rename to src/externals/mct/mpi-serial/list.h diff --git a/externals/mct/mpi-serial/listP.h b/src/externals/mct/mpi-serial/listP.h similarity index 100% rename from externals/mct/mpi-serial/listP.h rename to src/externals/mct/mpi-serial/listP.h diff --git a/externals/mct/mpi-serial/listops.h b/src/externals/mct/mpi-serial/listops.h similarity index 100% rename from externals/mct/mpi-serial/listops.h rename to src/externals/mct/mpi-serial/listops.h diff --git a/externals/mct/mpi-serial/m4/README b/src/externals/mct/mpi-serial/m4/README similarity index 100% rename from externals/mct/mpi-serial/m4/README rename to src/externals/mct/mpi-serial/m4/README diff --git a/externals/mct/mpi-serial/m4/ax_fc_version.m4 b/src/externals/mct/mpi-serial/m4/ax_fc_version.m4 similarity index 100% rename from externals/mct/mpi-serial/m4/ax_fc_version.m4 rename to src/externals/mct/mpi-serial/m4/ax_fc_version.m4 diff --git a/externals/mct/mpi-serial/mpi.c b/src/externals/mct/mpi-serial/mpi.c similarity index 100% rename from externals/mct/mpi-serial/mpi.c rename to src/externals/mct/mpi-serial/mpi.c diff --git a/externals/mct/mpi-serial/mpi.h b/src/externals/mct/mpi-serial/mpi.h similarity index 100% rename from externals/mct/mpi-serial/mpi.h rename to src/externals/mct/mpi-serial/mpi.h diff --git a/externals/mct/mpi-serial/mpiP.h b/src/externals/mct/mpi-serial/mpiP.h similarity index 100% rename from externals/mct/mpi-serial/mpiP.h rename to src/externals/mct/mpi-serial/mpiP.h diff --git a/externals/mct/mpi-serial/mpif.F90 b/src/externals/mct/mpi-serial/mpif.F90 similarity index 100% rename from externals/mct/mpi-serial/mpif.F90 rename to src/externals/mct/mpi-serial/mpif.F90 diff --git a/externals/mct/mpi-serial/mpif.h b/src/externals/mct/mpi-serial/mpif.h similarity index 100% rename from externals/mct/mpi-serial/mpif.h rename to src/externals/mct/mpi-serial/mpif.h diff --git a/externals/mct/mpi-serial/op.c b/src/externals/mct/mpi-serial/op.c similarity index 100% rename from externals/mct/mpi-serial/op.c rename to src/externals/mct/mpi-serial/op.c diff --git a/externals/mct/mpi-serial/pack.c b/src/externals/mct/mpi-serial/pack.c similarity index 100% rename from externals/mct/mpi-serial/pack.c rename to src/externals/mct/mpi-serial/pack.c diff --git a/externals/mct/mpi-serial/probe.c b/src/externals/mct/mpi-serial/probe.c similarity index 100% rename from externals/mct/mpi-serial/probe.c rename to src/externals/mct/mpi-serial/probe.c diff --git a/externals/mct/mpi-serial/protify.awk b/src/externals/mct/mpi-serial/protify.awk similarity index 100% rename from externals/mct/mpi-serial/protify.awk rename to src/externals/mct/mpi-serial/protify.awk diff --git a/externals/mct/mpi-serial/recv.c b/src/externals/mct/mpi-serial/recv.c similarity index 100% rename from externals/mct/mpi-serial/recv.c rename to src/externals/mct/mpi-serial/recv.c diff --git a/externals/mct/mpi-serial/req.c b/src/externals/mct/mpi-serial/req.c similarity index 100% rename from externals/mct/mpi-serial/req.c rename to src/externals/mct/mpi-serial/req.c diff --git a/externals/mct/mpi-serial/send.c b/src/externals/mct/mpi-serial/send.c similarity index 100% rename from externals/mct/mpi-serial/send.c rename to src/externals/mct/mpi-serial/send.c diff --git a/externals/mct/mpi-serial/tests/.gitignore b/src/externals/mct/mpi-serial/tests/.gitignore similarity index 100% rename from externals/mct/mpi-serial/tests/.gitignore rename to src/externals/mct/mpi-serial/tests/.gitignore diff --git a/externals/mct/mpi-serial/tests/Makefile b/src/externals/mct/mpi-serial/tests/Makefile similarity index 100% rename from externals/mct/mpi-serial/tests/Makefile rename to src/externals/mct/mpi-serial/tests/Makefile diff --git a/externals/mct/mpi-serial/tests/ctest.c b/src/externals/mct/mpi-serial/tests/ctest.c similarity index 100% rename from externals/mct/mpi-serial/tests/ctest.c rename to src/externals/mct/mpi-serial/tests/ctest.c diff --git a/externals/mct/mpi-serial/tests/ctest_old.c b/src/externals/mct/mpi-serial/tests/ctest_old.c similarity index 100% rename from externals/mct/mpi-serial/tests/ctest_old.c rename to src/externals/mct/mpi-serial/tests/ctest_old.c diff --git a/externals/mct/mpi-serial/tests/ftest.F90 b/src/externals/mct/mpi-serial/tests/ftest.F90 similarity index 100% rename from externals/mct/mpi-serial/tests/ftest.F90 rename to src/externals/mct/mpi-serial/tests/ftest.F90 diff --git a/externals/mct/mpi-serial/tests/ftest_internal.F90 b/src/externals/mct/mpi-serial/tests/ftest_internal.F90 similarity index 100% rename from externals/mct/mpi-serial/tests/ftest_internal.F90 rename to src/externals/mct/mpi-serial/tests/ftest_internal.F90 diff --git a/externals/mct/mpi-serial/tests/ftest_old.F90 b/src/externals/mct/mpi-serial/tests/ftest_old.F90 similarity index 100% rename from externals/mct/mpi-serial/tests/ftest_old.F90 rename to src/externals/mct/mpi-serial/tests/ftest_old.F90 diff --git a/externals/mct/mpi-serial/time.c b/src/externals/mct/mpi-serial/time.c similarity index 100% rename from externals/mct/mpi-serial/time.c rename to src/externals/mct/mpi-serial/time.c diff --git a/externals/mct/mpi-serial/type.c b/src/externals/mct/mpi-serial/type.c similarity index 100% rename from externals/mct/mpi-serial/type.c rename to src/externals/mct/mpi-serial/type.c diff --git a/externals/mct/mpi-serial/type.h b/src/externals/mct/mpi-serial/type.h similarity index 100% rename from externals/mct/mpi-serial/type.h rename to src/externals/mct/mpi-serial/type.h diff --git a/externals/mct/mpi-serial/type_const.c b/src/externals/mct/mpi-serial/type_const.c similarity index 100% rename from externals/mct/mpi-serial/type_const.c rename to src/externals/mct/mpi-serial/type_const.c diff --git a/externals/mct/protex/protex b/src/externals/mct/protex/protex similarity index 100% rename from externals/mct/protex/protex rename to src/externals/mct/protex/protex diff --git a/externals/mct/testsystem/Makefile b/src/externals/mct/testsystem/Makefile similarity index 100% rename from externals/mct/testsystem/Makefile rename to src/externals/mct/testsystem/Makefile diff --git a/externals/mct/testsystem/testall/.gitignore b/src/externals/mct/testsystem/testall/.gitignore similarity index 100% rename from externals/mct/testsystem/testall/.gitignore rename to src/externals/mct/testsystem/testall/.gitignore diff --git a/externals/mct/testsystem/testall/Makefile b/src/externals/mct/testsystem/testall/Makefile similarity index 100% rename from externals/mct/testsystem/testall/Makefile rename to src/externals/mct/testsystem/testall/Makefile diff --git a/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 b/src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 similarity index 100% rename from externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 rename to src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 diff --git a/externals/mct/testsystem/testall/UNTESTED b/src/externals/mct/testsystem/testall/UNTESTED similarity index 100% rename from externals/mct/testsystem/testall/UNTESTED rename to src/externals/mct/testsystem/testall/UNTESTED diff --git a/externals/mct/testsystem/testall/ccm.F90 b/src/externals/mct/testsystem/testall/ccm.F90 similarity index 100% rename from externals/mct/testsystem/testall/ccm.F90 rename to src/externals/mct/testsystem/testall/ccm.F90 diff --git a/externals/mct/testsystem/testall/convertPOPT.F90 b/src/externals/mct/testsystem/testall/convertPOPT.F90 similarity index 100% rename from externals/mct/testsystem/testall/convertPOPT.F90 rename to src/externals/mct/testsystem/testall/convertPOPT.F90 diff --git a/externals/mct/testsystem/testall/convertgauss.F90 b/src/externals/mct/testsystem/testall/convertgauss.F90 similarity index 100% rename from externals/mct/testsystem/testall/convertgauss.F90 rename to src/externals/mct/testsystem/testall/convertgauss.F90 diff --git a/externals/mct/testsystem/testall/cpl.F90 b/src/externals/mct/testsystem/testall/cpl.F90 similarity index 100% rename from externals/mct/testsystem/testall/cpl.F90 rename to src/externals/mct/testsystem/testall/cpl.F90 diff --git a/externals/mct/testsystem/testall/job.ut-all.jaguar b/src/externals/mct/testsystem/testall/job.ut-all.jaguar similarity index 100% rename from externals/mct/testsystem/testall/job.ut-all.jaguar rename to src/externals/mct/testsystem/testall/job.ut-all.jaguar diff --git a/externals/mct/testsystem/testall/m_ACTEST.F90 b/src/externals/mct/testsystem/testall/m_ACTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_ACTEST.F90 rename to src/externals/mct/testsystem/testall/m_ACTEST.F90 diff --git a/externals/mct/testsystem/testall/m_AVTEST.F90 b/src/externals/mct/testsystem/testall/m_AVTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_AVTEST.F90 rename to src/externals/mct/testsystem/testall/m_AVTEST.F90 diff --git a/externals/mct/testsystem/testall/m_GGRIDTEST.F90 b/src/externals/mct/testsystem/testall/m_GGRIDTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_GGRIDTEST.F90 rename to src/externals/mct/testsystem/testall/m_GGRIDTEST.F90 diff --git a/externals/mct/testsystem/testall/m_GMAPTEST.F90 b/src/externals/mct/testsystem/testall/m_GMAPTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_GMAPTEST.F90 rename to src/externals/mct/testsystem/testall/m_GMAPTEST.F90 diff --git a/externals/mct/testsystem/testall/m_GSMAPTEST.F90 b/src/externals/mct/testsystem/testall/m_GSMAPTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_GSMAPTEST.F90 rename to src/externals/mct/testsystem/testall/m_GSMAPTEST.F90 diff --git a/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 b/src/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 rename to src/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 diff --git a/externals/mct/testsystem/testall/m_ROUTERTEST.F90 b/src/externals/mct/testsystem/testall/m_ROUTERTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_ROUTERTEST.F90 rename to src/externals/mct/testsystem/testall/m_ROUTERTEST.F90 diff --git a/externals/mct/testsystem/testall/m_SMATTEST.F90 b/src/externals/mct/testsystem/testall/m_SMATTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_SMATTEST.F90 rename to src/externals/mct/testsystem/testall/m_SMATTEST.F90 diff --git a/externals/mct/testsystem/testall/master.F90 b/src/externals/mct/testsystem/testall/master.F90 similarity index 100% rename from externals/mct/testsystem/testall/master.F90 rename to src/externals/mct/testsystem/testall/master.F90 diff --git a/externals/mct/testsystem/testall/mph.F90 b/src/externals/mct/testsystem/testall/mph.F90 similarity index 100% rename from externals/mct/testsystem/testall/mph.F90 rename to src/externals/mct/testsystem/testall/mph.F90 diff --git a/externals/mct/testsystem/testall/pop.F90 b/src/externals/mct/testsystem/testall/pop.F90 similarity index 100% rename from externals/mct/testsystem/testall/pop.F90 rename to src/externals/mct/testsystem/testall/pop.F90 diff --git a/externals/mct/testsystem/testall/processors_map.in b/src/externals/mct/testsystem/testall/processors_map.in similarity index 100% rename from externals/mct/testsystem/testall/processors_map.in rename to src/externals/mct/testsystem/testall/processors_map.in diff --git a/externals/mct/testsystem/testall/script.jag b/src/externals/mct/testsystem/testall/script.jag similarity index 100% rename from externals/mct/testsystem/testall/script.jag rename to src/externals/mct/testsystem/testall/script.jag diff --git a/externals/mct/testsystem/testall/ut_SparseMatrix.rc b/src/externals/mct/testsystem/testall/ut_SparseMatrix.rc similarity index 100% rename from externals/mct/testsystem/testall/ut_SparseMatrix.rc rename to src/externals/mct/testsystem/testall/ut_SparseMatrix.rc diff --git a/externals/mct/testunit/.gitignore b/src/externals/mct/testunit/.gitignore similarity index 100% rename from externals/mct/testunit/.gitignore rename to src/externals/mct/testunit/.gitignore diff --git a/externals/mct/testunit/AttrVect_Test.F90 b/src/externals/mct/testunit/AttrVect_Test.F90 similarity index 100% rename from externals/mct/testunit/AttrVect_Test.F90 rename to src/externals/mct/testunit/AttrVect_Test.F90 diff --git a/externals/mct/testunit/Makefile b/src/externals/mct/testunit/Makefile similarity index 100% rename from externals/mct/testunit/Makefile rename to src/externals/mct/testunit/Makefile diff --git a/externals/mct/testunit/master.F90 b/src/externals/mct/testunit/master.F90 similarity index 100% rename from externals/mct/testunit/master.F90 rename to src/externals/mct/testunit/master.F90 diff --git a/externals/pio1/CMakeLists.txt b/src/externals/pio1/CMakeLists.txt similarity index 100% rename from externals/pio1/CMakeLists.txt rename to src/externals/pio1/CMakeLists.txt diff --git a/externals/pio1/ChangeLog b/src/externals/pio1/ChangeLog similarity index 100% rename from externals/pio1/ChangeLog rename to src/externals/pio1/ChangeLog diff --git a/externals/pio1/ChangeLog_template b/src/externals/pio1/ChangeLog_template similarity index 100% rename from externals/pio1/ChangeLog_template rename to src/externals/pio1/ChangeLog_template diff --git a/externals/pio1/Doxyfile b/src/externals/pio1/Doxyfile similarity index 100% rename from externals/pio1/Doxyfile rename to src/externals/pio1/Doxyfile diff --git a/externals/pio1/DoxygenLayout.xml b/src/externals/pio1/DoxygenLayout.xml similarity index 100% rename from externals/pio1/DoxygenLayout.xml rename to src/externals/pio1/DoxygenLayout.xml diff --git a/externals/pio1/SVN_EXTERNAL_DIRECTORIES b/src/externals/pio1/SVN_EXTERNAL_DIRECTORIES similarity index 100% rename from externals/pio1/SVN_EXTERNAL_DIRECTORIES rename to src/externals/pio1/SVN_EXTERNAL_DIRECTORIES diff --git a/externals/pio1/customdoxygen.css b/src/externals/pio1/customdoxygen.css similarity index 100% rename from externals/pio1/customdoxygen.css rename to src/externals/pio1/customdoxygen.css diff --git a/externals/pio1/doc/CAMexample.txt b/src/externals/pio1/doc/CAMexample.txt similarity index 100% rename from externals/pio1/doc/CAMexample.txt rename to src/externals/pio1/doc/CAMexample.txt diff --git a/externals/pio1/doc/Decomp.txt b/src/externals/pio1/doc/Decomp.txt similarity index 100% rename from externals/pio1/doc/Decomp.txt rename to src/externals/pio1/doc/Decomp.txt diff --git a/externals/pio1/doc/DoxygenLayout.xml b/src/externals/pio1/doc/DoxygenLayout.xml similarity index 100% rename from externals/pio1/doc/DoxygenLayout.xml rename to src/externals/pio1/doc/DoxygenLayout.xml diff --git a/externals/pio1/doc/Error.txt b/src/externals/pio1/doc/Error.txt similarity index 100% rename from externals/pio1/doc/Error.txt rename to src/externals/pio1/doc/Error.txt diff --git a/externals/pio1/doc/Examples.txt b/src/externals/pio1/doc/Examples.txt similarity index 100% rename from externals/pio1/doc/Examples.txt rename to src/externals/pio1/doc/Examples.txt diff --git a/externals/pio1/doc/GettingStarted.txt b/src/externals/pio1/doc/GettingStarted.txt similarity index 100% rename from externals/pio1/doc/GettingStarted.txt rename to src/externals/pio1/doc/GettingStarted.txt diff --git a/externals/pio1/doc/Installing.txt b/src/externals/pio1/doc/Installing.txt similarity index 100% rename from externals/pio1/doc/Installing.txt rename to src/externals/pio1/doc/Installing.txt diff --git a/externals/pio1/doc/api.txt b/src/externals/pio1/doc/api.txt similarity index 100% rename from externals/pio1/doc/api.txt rename to src/externals/pio1/doc/api.txt diff --git a/externals/pio1/doc/base.txt b/src/externals/pio1/doc/base.txt similarity index 100% rename from externals/pio1/doc/base.txt rename to src/externals/pio1/doc/base.txt diff --git a/externals/pio1/doc/example/errorhandle b/src/externals/pio1/doc/example/errorhandle similarity index 100% rename from externals/pio1/doc/example/errorhandle rename to src/externals/pio1/doc/example/errorhandle diff --git a/externals/pio1/doc/example/simple-bc b/src/externals/pio1/doc/example/simple-bc similarity index 100% rename from externals/pio1/doc/example/simple-bc rename to src/externals/pio1/doc/example/simple-bc diff --git a/externals/pio1/doc/example/simple-bc-rearr b/src/externals/pio1/doc/example/simple-bc-rearr similarity index 100% rename from externals/pio1/doc/example/simple-bc-rearr rename to src/externals/pio1/doc/example/simple-bc-rearr diff --git a/externals/pio1/doc/example/simple-bc-rearr-pe1 b/src/externals/pio1/doc/example/simple-bc-rearr-pe1 similarity index 100% rename from externals/pio1/doc/example/simple-bc-rearr-pe1 rename to src/externals/pio1/doc/example/simple-bc-rearr-pe1 diff --git a/externals/pio1/doc/example/simple-bc-rearr-pe2 b/src/externals/pio1/doc/example/simple-bc-rearr-pe2 similarity index 100% rename from externals/pio1/doc/example/simple-bc-rearr-pe2 rename to src/externals/pio1/doc/example/simple-bc-rearr-pe2 diff --git a/externals/pio1/doc/example/simple-dof b/src/externals/pio1/doc/example/simple-dof similarity index 100% rename from externals/pio1/doc/example/simple-dof rename to src/externals/pio1/doc/example/simple-dof diff --git a/externals/pio1/doc/example/simple-dof-rearr b/src/externals/pio1/doc/example/simple-dof-rearr similarity index 100% rename from externals/pio1/doc/example/simple-dof-rearr rename to src/externals/pio1/doc/example/simple-dof-rearr diff --git a/externals/pio1/doc/faq.txt b/src/externals/pio1/doc/faq.txt similarity index 100% rename from externals/pio1/doc/faq.txt rename to src/externals/pio1/doc/faq.txt diff --git a/externals/pio1/doc/footer.html b/src/externals/pio1/doc/footer.html similarity index 100% rename from externals/pio1/doc/footer.html rename to src/externals/pio1/doc/footer.html diff --git a/externals/pio1/doc/header.html b/src/externals/pio1/doc/header.html similarity index 100% rename from externals/pio1/doc/header.html rename to src/externals/pio1/doc/header.html diff --git a/externals/pio1/doc/images/baseimage.graffle b/src/externals/pio1/doc/images/baseimage.graffle similarity index 100% rename from externals/pio1/doc/images/baseimage.graffle rename to src/externals/pio1/doc/images/baseimage.graffle diff --git a/externals/pio1/doc/images/block-cyclic-rearr.eps b/src/externals/pio1/doc/images/block-cyclic-rearr.eps similarity index 100% rename from externals/pio1/doc/images/block-cyclic-rearr.eps rename to src/externals/pio1/doc/images/block-cyclic-rearr.eps diff --git a/externals/pio1/doc/images/block-cyclic-rearr.graffle b/src/externals/pio1/doc/images/block-cyclic-rearr.graffle similarity index 100% rename from externals/pio1/doc/images/block-cyclic-rearr.graffle rename to src/externals/pio1/doc/images/block-cyclic-rearr.graffle diff --git a/externals/pio1/doc/images/block-cyclic-rearr.png b/src/externals/pio1/doc/images/block-cyclic-rearr.png similarity index 100% rename from externals/pio1/doc/images/block-cyclic-rearr.png rename to src/externals/pio1/doc/images/block-cyclic-rearr.png diff --git a/externals/pio1/doc/images/block-cyclic.eps b/src/externals/pio1/doc/images/block-cyclic.eps similarity index 100% rename from externals/pio1/doc/images/block-cyclic.eps rename to src/externals/pio1/doc/images/block-cyclic.eps diff --git a/externals/pio1/doc/images/block-cyclic.graffle b/src/externals/pio1/doc/images/block-cyclic.graffle similarity index 100% rename from externals/pio1/doc/images/block-cyclic.graffle rename to src/externals/pio1/doc/images/block-cyclic.graffle diff --git a/externals/pio1/doc/images/block-cyclic.png b/src/externals/pio1/doc/images/block-cyclic.png similarity index 100% rename from externals/pio1/doc/images/block-cyclic.png rename to src/externals/pio1/doc/images/block-cyclic.png diff --git a/externals/pio1/doc/images/dof-rearr.eps b/src/externals/pio1/doc/images/dof-rearr.eps similarity index 100% rename from externals/pio1/doc/images/dof-rearr.eps rename to src/externals/pio1/doc/images/dof-rearr.eps diff --git a/externals/pio1/doc/images/dof-rearr.graffle b/src/externals/pio1/doc/images/dof-rearr.graffle similarity index 100% rename from externals/pio1/doc/images/dof-rearr.graffle rename to src/externals/pio1/doc/images/dof-rearr.graffle diff --git a/externals/pio1/doc/images/dof-rearr.png b/src/externals/pio1/doc/images/dof-rearr.png similarity index 100% rename from externals/pio1/doc/images/dof-rearr.png rename to src/externals/pio1/doc/images/dof-rearr.png diff --git a/externals/pio1/doc/images/dof.eps b/src/externals/pio1/doc/images/dof.eps similarity index 100% rename from externals/pio1/doc/images/dof.eps rename to src/externals/pio1/doc/images/dof.eps diff --git a/externals/pio1/doc/images/dof.graffle b/src/externals/pio1/doc/images/dof.graffle similarity index 100% rename from externals/pio1/doc/images/dof.graffle rename to src/externals/pio1/doc/images/dof.graffle diff --git a/externals/pio1/doc/images/dof.png b/src/externals/pio1/doc/images/dof.png similarity index 100% rename from externals/pio1/doc/images/dof.png rename to src/externals/pio1/doc/images/dof.png diff --git a/externals/pio1/doc/testpio_example.txt b/src/externals/pio1/doc/testpio_example.txt similarity index 100% rename from externals/pio1/doc/testpio_example.txt rename to src/externals/pio1/doc/testpio_example.txt diff --git a/externals/pio1/doxygen.sty b/src/externals/pio1/doxygen.sty similarity index 100% rename from externals/pio1/doxygen.sty rename to src/externals/pio1/doxygen.sty diff --git a/externals/pio1/pio/CMakeLists.txt b/src/externals/pio1/pio/CMakeLists.txt similarity index 100% rename from externals/pio1/pio/CMakeLists.txt rename to src/externals/pio1/pio/CMakeLists.txt diff --git a/externals/pio1/pio/C_interface_mod.F90 b/src/externals/pio1/pio/C_interface_mod.F90 similarity index 100% rename from externals/pio1/pio/C_interface_mod.F90 rename to src/externals/pio1/pio/C_interface_mod.F90 diff --git a/externals/pio1/pio/Makefile.conf.in b/src/externals/pio1/pio/Makefile.conf.in similarity index 100% rename from externals/pio1/pio/Makefile.conf.in rename to src/externals/pio1/pio/Makefile.conf.in diff --git a/externals/pio1/pio/README.config b/src/externals/pio1/pio/README.config similarity index 100% rename from externals/pio1/pio/README.config rename to src/externals/pio1/pio/README.config diff --git a/externals/pio1/pio/alloc_mod.F90.in b/src/externals/pio1/pio/alloc_mod.F90.in similarity index 100% rename from externals/pio1/pio/alloc_mod.F90.in rename to src/externals/pio1/pio/alloc_mod.F90.in diff --git a/externals/pio1/pio/box_rearrange.F90.in b/src/externals/pio1/pio/box_rearrange.F90.in similarity index 100% rename from externals/pio1/pio/box_rearrange.F90.in rename to src/externals/pio1/pio/box_rearrange.F90.in diff --git a/externals/pio1/pio/calcdecomp.F90 b/src/externals/pio1/pio/calcdecomp.F90 similarity index 100% rename from externals/pio1/pio/calcdecomp.F90 rename to src/externals/pio1/pio/calcdecomp.F90 diff --git a/externals/pio1/pio/calcdisplace_mod.F90 b/src/externals/pio1/pio/calcdisplace_mod.F90 similarity index 100% rename from externals/pio1/pio/calcdisplace_mod.F90 rename to src/externals/pio1/pio/calcdisplace_mod.F90 diff --git a/externals/pio1/pio/config.h.in b/src/externals/pio1/pio/config.h.in similarity index 100% rename from externals/pio1/pio/config.h.in rename to src/externals/pio1/pio/config.h.in diff --git a/externals/pio1/pio/dtypes.h b/src/externals/pio1/pio/dtypes.h similarity index 100% rename from externals/pio1/pio/dtypes.h rename to src/externals/pio1/pio/dtypes.h diff --git a/externals/pio1/pio/fdepends.awk b/src/externals/pio1/pio/fdepends.awk similarity index 100% rename from externals/pio1/pio/fdepends.awk rename to src/externals/pio1/pio/fdepends.awk diff --git a/externals/pio1/pio/iompi_mod.F90.in b/src/externals/pio1/pio/iompi_mod.F90.in similarity index 100% rename from externals/pio1/pio/iompi_mod.F90.in rename to src/externals/pio1/pio/iompi_mod.F90.in diff --git a/externals/pio1/pio/ionf_mod.F90 b/src/externals/pio1/pio/ionf_mod.F90 similarity index 100% rename from externals/pio1/pio/ionf_mod.F90 rename to src/externals/pio1/pio/ionf_mod.F90 diff --git a/externals/pio1/pio/nf_mod.F90 b/src/externals/pio1/pio/nf_mod.F90 similarity index 100% rename from externals/pio1/pio/nf_mod.F90 rename to src/externals/pio1/pio/nf_mod.F90 diff --git a/externals/pio1/pio/pio.F90 b/src/externals/pio1/pio/pio.F90 similarity index 100% rename from externals/pio1/pio/pio.F90 rename to src/externals/pio1/pio/pio.F90 diff --git a/externals/pio1/pio/pio_kinds.F90 b/src/externals/pio1/pio/pio_kinds.F90 similarity index 100% rename from externals/pio1/pio/pio_kinds.F90 rename to src/externals/pio1/pio/pio_kinds.F90 diff --git a/externals/pio1/pio/pio_mpi_utils.F90 b/src/externals/pio1/pio/pio_mpi_utils.F90 similarity index 100% rename from externals/pio1/pio/pio_mpi_utils.F90 rename to src/externals/pio1/pio/pio_mpi_utils.F90 diff --git a/externals/pio1/pio/pio_msg_callbacks.F90 b/src/externals/pio1/pio/pio_msg_callbacks.F90 similarity index 100% rename from externals/pio1/pio/pio_msg_callbacks.F90 rename to src/externals/pio1/pio/pio_msg_callbacks.F90 diff --git a/externals/pio1/pio/pio_msg_getput_callbacks.F90.in b/src/externals/pio1/pio/pio_msg_getput_callbacks.F90.in similarity index 100% rename from externals/pio1/pio/pio_msg_getput_callbacks.F90.in rename to src/externals/pio1/pio/pio_msg_getput_callbacks.F90.in diff --git a/externals/pio1/pio/pio_msg_mod.F90 b/src/externals/pio1/pio/pio_msg_mod.F90 similarity index 100% rename from externals/pio1/pio/pio_msg_mod.F90 rename to src/externals/pio1/pio/pio_msg_mod.F90 diff --git a/externals/pio1/pio/pio_nf_utils.F90 b/src/externals/pio1/pio/pio_nf_utils.F90 similarity index 100% rename from externals/pio1/pio/pio_nf_utils.F90 rename to src/externals/pio1/pio/pio_nf_utils.F90 diff --git a/externals/pio1/pio/pio_spmd_utils.F90.in b/src/externals/pio1/pio/pio_spmd_utils.F90.in similarity index 100% rename from externals/pio1/pio/pio_spmd_utils.F90.in rename to src/externals/pio1/pio/pio_spmd_utils.F90.in diff --git a/externals/pio1/pio/pio_support.F90 b/src/externals/pio1/pio/pio_support.F90 similarity index 100% rename from externals/pio1/pio/pio_support.F90 rename to src/externals/pio1/pio/pio_support.F90 diff --git a/externals/pio1/pio/pio_types.F90 b/src/externals/pio1/pio/pio_types.F90 similarity index 100% rename from externals/pio1/pio/pio_types.F90 rename to src/externals/pio1/pio/pio_types.F90 diff --git a/externals/pio1/pio/pio_utils.F90 b/src/externals/pio1/pio/pio_utils.F90 similarity index 100% rename from externals/pio1/pio/pio_utils.F90 rename to src/externals/pio1/pio/pio_utils.F90 diff --git a/externals/pio1/pio/piodarray.F90.in b/src/externals/pio1/pio/piodarray.F90.in similarity index 100% rename from externals/pio1/pio/piodarray.F90.in rename to src/externals/pio1/pio/piodarray.F90.in diff --git a/externals/pio1/pio/piolib_mod.F90 b/src/externals/pio1/pio/piolib_mod.F90 similarity index 100% rename from externals/pio1/pio/piolib_mod.F90 rename to src/externals/pio1/pio/piolib_mod.F90 diff --git a/externals/pio1/pio/pionfatt_mod.F90.in b/src/externals/pio1/pio/pionfatt_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfatt_mod.F90.in rename to src/externals/pio1/pio/pionfatt_mod.F90.in diff --git a/externals/pio1/pio/pionfget_mod.F90.in b/src/externals/pio1/pio/pionfget_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfget_mod.F90.in rename to src/externals/pio1/pio/pionfget_mod.F90.in diff --git a/externals/pio1/pio/pionfput_mod.F90.in b/src/externals/pio1/pio/pionfput_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfput_mod.F90.in rename to src/externals/pio1/pio/pionfput_mod.F90.in diff --git a/externals/pio1/pio/pionfread_mod.F90.in b/src/externals/pio1/pio/pionfread_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfread_mod.F90.in rename to src/externals/pio1/pio/pionfread_mod.F90.in diff --git a/externals/pio1/pio/pionfwrite_mod.F90.in b/src/externals/pio1/pio/pionfwrite_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfwrite_mod.F90.in rename to src/externals/pio1/pio/pionfwrite_mod.F90.in diff --git a/externals/pio1/pio/piovdc.F90 b/src/externals/pio1/pio/piovdc.F90 similarity index 100% rename from externals/pio1/pio/piovdc.F90 rename to src/externals/pio1/pio/piovdc.F90 diff --git a/externals/pio1/pio/rearr_options.h b/src/externals/pio1/pio/rearr_options.h similarity index 100% rename from externals/pio1/pio/rearr_options.h rename to src/externals/pio1/pio/rearr_options.h diff --git a/externals/pio1/pio/rearrange.F90.in b/src/externals/pio1/pio/rearrange.F90.in similarity index 100% rename from externals/pio1/pio/rearrange.F90.in rename to src/externals/pio1/pio/rearrange.F90.in diff --git a/externals/pio1/pio/topology.c b/src/externals/pio1/pio/topology.c similarity index 100% rename from externals/pio1/pio/topology.c rename to src/externals/pio1/pio/topology.c diff --git a/externals/pio1/scripts/Utils.pm b/src/externals/pio1/scripts/Utils.pm similarity index 100% rename from externals/pio1/scripts/Utils.pm rename to src/externals/pio1/scripts/Utils.pm diff --git a/externals/pio1/scripts/config.pl b/src/externals/pio1/scripts/config.pl similarity index 100% rename from externals/pio1/scripts/config.pl rename to src/externals/pio1/scripts/config.pl diff --git a/externals/pio1/scripts/testpio_yellowstone.pl b/src/externals/pio1/scripts/testpio_yellowstone.pl similarity index 100% rename from externals/pio1/scripts/testpio_yellowstone.pl rename to src/externals/pio1/scripts/testpio_yellowstone.pl diff --git a/externals/pio1/tests/testpio/CAM05.csh b/src/externals/pio1/tests/testpio/CAM05.csh similarity index 100% rename from externals/pio1/tests/testpio/CAM05.csh rename to src/externals/pio1/tests/testpio/CAM05.csh diff --git a/externals/pio1/tests/testpio/CMakeLists.txt b/src/externals/pio1/tests/testpio/CMakeLists.txt similarity index 100% rename from externals/pio1/tests/testpio/CMakeLists.txt rename to src/externals/pio1/tests/testpio/CMakeLists.txt diff --git a/externals/pio1/tests/testpio/MPASA30km.csh b/src/externals/pio1/tests/testpio/MPASA30km.csh similarity index 100% rename from externals/pio1/tests/testpio/MPASA30km.csh rename to src/externals/pio1/tests/testpio/MPASA30km.csh diff --git a/externals/pio1/tests/testpio/MPASA60km.csh b/src/externals/pio1/tests/testpio/MPASA60km.csh similarity index 100% rename from externals/pio1/tests/testpio/MPASA60km.csh rename to src/externals/pio1/tests/testpio/MPASA60km.csh diff --git a/externals/pio1/tests/testpio/POPB.csh b/src/externals/pio1/tests/testpio/POPB.csh similarity index 100% rename from externals/pio1/tests/testpio/POPB.csh rename to src/externals/pio1/tests/testpio/POPB.csh diff --git a/externals/pio1/tests/testpio/POPC.csh b/src/externals/pio1/tests/testpio/POPC.csh similarity index 100% rename from externals/pio1/tests/testpio/POPC.csh rename to src/externals/pio1/tests/testpio/POPC.csh diff --git a/externals/pio1/tests/testpio/POPD.csh b/src/externals/pio1/tests/testpio/POPD.csh similarity index 100% rename from externals/pio1/tests/testpio/POPD.csh rename to src/externals/pio1/tests/testpio/POPD.csh diff --git a/externals/pio1/tests/testpio/POPDv0.csh b/src/externals/pio1/tests/testpio/POPDv0.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv0.csh rename to src/externals/pio1/tests/testpio/POPDv0.csh diff --git a/externals/pio1/tests/testpio/POPDv1.csh b/src/externals/pio1/tests/testpio/POPDv1.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv1.csh rename to src/externals/pio1/tests/testpio/POPDv1.csh diff --git a/externals/pio1/tests/testpio/POPDv2.csh b/src/externals/pio1/tests/testpio/POPDv2.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv2.csh rename to src/externals/pio1/tests/testpio/POPDv2.csh diff --git a/externals/pio1/tests/testpio/POPDv3.csh b/src/externals/pio1/tests/testpio/POPDv3.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv3.csh rename to src/externals/pio1/tests/testpio/POPDv3.csh diff --git a/externals/pio1/tests/testpio/POPDv4.csh b/src/externals/pio1/tests/testpio/POPDv4.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv4.csh rename to src/externals/pio1/tests/testpio/POPDv4.csh diff --git a/externals/pio1/tests/testpio/POPDv5.csh b/src/externals/pio1/tests/testpio/POPDv5.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv5.csh rename to src/externals/pio1/tests/testpio/POPDv5.csh diff --git a/externals/pio1/tests/testpio/README.testpio b/src/externals/pio1/tests/testpio/README.testpio similarity index 100% rename from externals/pio1/tests/testpio/README.testpio rename to src/externals/pio1/tests/testpio/README.testpio diff --git a/externals/pio1/tests/testpio/WRFB.csh b/src/externals/pio1/tests/testpio/WRFB.csh similarity index 100% rename from externals/pio1/tests/testpio/WRFB.csh rename to src/externals/pio1/tests/testpio/WRFB.csh diff --git a/externals/pio1/tests/testpio/build_defaults.xml b/src/externals/pio1/tests/testpio/build_defaults.xml similarity index 100% rename from externals/pio1/tests/testpio/build_defaults.xml rename to src/externals/pio1/tests/testpio/build_defaults.xml diff --git a/externals/pio1/tests/testpio/check_mod.F90 b/src/externals/pio1/tests/testpio/check_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/check_mod.F90 rename to src/externals/pio1/tests/testpio/check_mod.F90 diff --git a/externals/pio1/tests/testpio/config_bench.xml b/src/externals/pio1/tests/testpio/config_bench.xml similarity index 100% rename from externals/pio1/tests/testpio/config_bench.xml rename to src/externals/pio1/tests/testpio/config_bench.xml diff --git a/externals/pio1/tests/testpio/fdepends.awk b/src/externals/pio1/tests/testpio/fdepends.awk similarity index 100% rename from externals/pio1/tests/testpio/fdepends.awk rename to src/externals/pio1/tests/testpio/fdepends.awk diff --git a/externals/pio1/tests/testpio/gdecomp_mod.F90 b/src/externals/pio1/tests/testpio/gdecomp_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/gdecomp_mod.F90 rename to src/externals/pio1/tests/testpio/gdecomp_mod.F90 diff --git a/externals/pio1/tests/testpio/kinds_mod.F90 b/src/externals/pio1/tests/testpio/kinds_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/kinds_mod.F90 rename to src/externals/pio1/tests/testpio/kinds_mod.F90 diff --git a/externals/pio1/tests/testpio/kraken.128.csh b/src/externals/pio1/tests/testpio/kraken.128.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.128.csh rename to src/externals/pio1/tests/testpio/kraken.128.csh diff --git a/externals/pio1/tests/testpio/kraken.1K.csh b/src/externals/pio1/tests/testpio/kraken.1K.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.1K.csh rename to src/externals/pio1/tests/testpio/kraken.1K.csh diff --git a/externals/pio1/tests/testpio/kraken.256.csh b/src/externals/pio1/tests/testpio/kraken.256.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.256.csh rename to src/externals/pio1/tests/testpio/kraken.256.csh diff --git a/externals/pio1/tests/testpio/kraken.512.csh b/src/externals/pio1/tests/testpio/kraken.512.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.512.csh rename to src/externals/pio1/tests/testpio/kraken.512.csh diff --git a/externals/pio1/tests/testpio/kraken.64.csh b/src/externals/pio1/tests/testpio/kraken.64.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.64.csh rename to src/externals/pio1/tests/testpio/kraken.64.csh diff --git a/externals/pio1/tests/testpio/namelist_mod.F90 b/src/externals/pio1/tests/testpio/namelist_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/namelist_mod.F90 rename to src/externals/pio1/tests/testpio/namelist_mod.F90 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.apb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.apb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.apb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.apb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.asb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.asb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.asb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.asb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.asb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.asb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.asb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.asb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b09 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b09 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b09 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b09 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b10 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b10 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b10 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b10 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b11 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b11 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b11 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b11 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b12 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b12 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b12 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b12 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b13 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b13 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b13 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b13 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bn01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bn01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bn01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bn01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bn02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bn02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bn02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bn02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bn03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bn03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bn03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bn03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4n01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4n02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4n03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pn01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pn01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pn01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pn01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pn02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pn02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pn02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pn02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pn03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pn03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pn03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pn03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sn01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sn01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sn01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sn01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sn02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sn02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sn02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sn02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sn03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sn03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sn03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sn03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.wr01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.wr01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.wr01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.wr01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.wr02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.wr02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.wr02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.wr02 diff --git a/externals/pio1/tests/testpio/perl5lib/ChangeLog b/src/externals/pio1/tests/testpio/perl5lib/ChangeLog similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/ChangeLog rename to src/externals/pio1/tests/testpio/perl5lib/ChangeLog diff --git a/externals/pio1/tests/testpio/perl5lib/README b/src/externals/pio1/tests/testpio/perl5lib/README similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/README rename to src/externals/pio1/tests/testpio/perl5lib/README diff --git a/externals/pio1/tests/testpio/perl5lib/XML/Changes b/src/externals/pio1/tests/testpio/perl5lib/XML/Changes similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/Changes rename to src/externals/pio1/tests/testpio/perl5lib/XML/Changes diff --git a/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm b/src/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/Lite.pm rename to src/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm diff --git a/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm b/src/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm rename to src/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm diff --git a/externals/pio1/tests/testpio/perl5lib/XML/README b/src/externals/pio1/tests/testpio/perl5lib/XML/README similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/README rename to src/externals/pio1/tests/testpio/perl5lib/XML/README diff --git a/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 b/src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 rename to src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 diff --git a/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 b/src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 rename to src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 diff --git a/externals/pio1/tests/testpio/test.csh b/src/externals/pio1/tests/testpio/test.csh similarity index 100% rename from externals/pio1/tests/testpio/test.csh rename to src/externals/pio1/tests/testpio/test.csh diff --git a/externals/pio1/tests/testpio/test_lib.F90 b/src/externals/pio1/tests/testpio/test_lib.F90 similarity index 100% rename from externals/pio1/tests/testpio/test_lib.F90 rename to src/externals/pio1/tests/testpio/test_lib.F90 diff --git a/externals/pio1/tests/testpio/testdecomp.F90 b/src/externals/pio1/tests/testpio/testdecomp.F90 similarity index 100% rename from externals/pio1/tests/testpio/testdecomp.F90 rename to src/externals/pio1/tests/testpio/testdecomp.F90 diff --git a/externals/pio1/tests/testpio/testdecomp.bluefire.run b/src/externals/pio1/tests/testpio/testdecomp.bluefire.run similarity index 100% rename from externals/pio1/tests/testpio/testdecomp.bluefire.run rename to src/externals/pio1/tests/testpio/testdecomp.bluefire.run diff --git a/externals/pio1/tests/testpio/testdecomp_in b/src/externals/pio1/tests/testpio/testdecomp_in similarity index 100% rename from externals/pio1/tests/testpio/testdecomp_in rename to src/externals/pio1/tests/testpio/testdecomp_in diff --git a/externals/pio1/tests/testpio/testpio.F90 b/src/externals/pio1/tests/testpio/testpio.F90 similarity index 100% rename from externals/pio1/tests/testpio/testpio.F90 rename to src/externals/pio1/tests/testpio/testpio.F90 diff --git a/externals/pio1/tests/testpio/testpio_bench.pl b/src/externals/pio1/tests/testpio/testpio_bench.pl similarity index 100% rename from externals/pio1/tests/testpio/testpio_bench.pl rename to src/externals/pio1/tests/testpio/testpio_bench.pl diff --git a/externals/pio1/tests/testpio/testpio_build.pl b/src/externals/pio1/tests/testpio/testpio_build.pl similarity index 100% rename from externals/pio1/tests/testpio/testpio_build.pl rename to src/externals/pio1/tests/testpio/testpio_build.pl diff --git a/externals/pio1/tests/testpio/testpio_run.pl b/src/externals/pio1/tests/testpio/testpio_run.pl similarity index 100% rename from externals/pio1/tests/testpio/testpio_run.pl rename to src/externals/pio1/tests/testpio/testpio_run.pl diff --git a/externals/pio1/tests/testpio/utils_mod.F90 b/src/externals/pio1/tests/testpio/utils_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/utils_mod.F90 rename to src/externals/pio1/tests/testpio/utils_mod.F90 diff --git a/externals/pio1/tests/testpio/ystest.sh b/src/externals/pio1/tests/testpio/ystest.sh similarity index 100% rename from externals/pio1/tests/testpio/ystest.sh rename to src/externals/pio1/tests/testpio/ystest.sh diff --git a/externals/pio1/tests/unittests/CMakeLists.txt b/src/externals/pio1/tests/unittests/CMakeLists.txt similarity index 100% rename from externals/pio1/tests/unittests/CMakeLists.txt rename to src/externals/pio1/tests/unittests/CMakeLists.txt diff --git a/externals/pio1/tests/unittests/Levy_Notes b/src/externals/pio1/tests/unittests/Levy_Notes similarity index 100% rename from externals/pio1/tests/unittests/Levy_Notes rename to src/externals/pio1/tests/unittests/Levy_Notes diff --git a/externals/pio1/tests/unittests/README b/src/externals/pio1/tests/unittests/README similarity index 100% rename from externals/pio1/tests/unittests/README rename to src/externals/pio1/tests/unittests/README diff --git a/externals/pio1/tests/unittests/basic_tests.F90 b/src/externals/pio1/tests/unittests/basic_tests.F90 similarity index 100% rename from externals/pio1/tests/unittests/basic_tests.F90 rename to src/externals/pio1/tests/unittests/basic_tests.F90 diff --git a/externals/pio1/tests/unittests/driver.F90 b/src/externals/pio1/tests/unittests/driver.F90 similarity index 100% rename from externals/pio1/tests/unittests/driver.F90 rename to src/externals/pio1/tests/unittests/driver.F90 diff --git a/externals/pio1/tests/unittests/global_vars.F90 b/src/externals/pio1/tests/unittests/global_vars.F90 similarity index 100% rename from externals/pio1/tests/unittests/global_vars.F90 rename to src/externals/pio1/tests/unittests/global_vars.F90 diff --git a/externals/pio1/tests/unittests/input.nl b/src/externals/pio1/tests/unittests/input.nl similarity index 100% rename from externals/pio1/tests/unittests/input.nl rename to src/externals/pio1/tests/unittests/input.nl diff --git a/externals/pio1/tests/unittests/nc_set_log_level2.c b/src/externals/pio1/tests/unittests/nc_set_log_level2.c similarity index 100% rename from externals/pio1/tests/unittests/nc_set_log_level2.c rename to src/externals/pio1/tests/unittests/nc_set_log_level2.c diff --git a/externals/pio1/tests/unittests/ncdf_tests.F90 b/src/externals/pio1/tests/unittests/ncdf_tests.F90 similarity index 100% rename from externals/pio1/tests/unittests/ncdf_tests.F90 rename to src/externals/pio1/tests/unittests/ncdf_tests.F90 diff --git a/externals/pio1/tests/unittests/not_netcdf.ieee b/src/externals/pio1/tests/unittests/not_netcdf.ieee similarity index 100% rename from externals/pio1/tests/unittests/not_netcdf.ieee rename to src/externals/pio1/tests/unittests/not_netcdf.ieee diff --git a/externals/pio1/timing/CMakeLists.txt b/src/externals/pio1/timing/CMakeLists.txt similarity index 100% rename from externals/pio1/timing/CMakeLists.txt rename to src/externals/pio1/timing/CMakeLists.txt diff --git a/src/externals/pio1/timing/COPYING b/src/externals/pio1/timing/COPYING new file mode 100644 index 00000000000..324ce86b24c --- /dev/null +++ b/src/externals/pio1/timing/COPYING @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software for any noncommercial purposes without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the following +conditions: The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. Any +commercial use (including sale) of the software, and derivative development +towards commercial use, requires written permission of the copyright +holder. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES +OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/externals/pio1/timing/ChangeLog b/src/externals/pio1/timing/ChangeLog similarity index 100% rename from externals/pio1/timing/ChangeLog rename to src/externals/pio1/timing/ChangeLog diff --git a/externals/pio1/timing/GPTLget_memusage.c b/src/externals/pio1/timing/GPTLget_memusage.c similarity index 100% rename from externals/pio1/timing/GPTLget_memusage.c rename to src/externals/pio1/timing/GPTLget_memusage.c diff --git a/externals/pio1/timing/GPTLprint_memusage.c b/src/externals/pio1/timing/GPTLprint_memusage.c similarity index 100% rename from externals/pio1/timing/GPTLprint_memusage.c rename to src/externals/pio1/timing/GPTLprint_memusage.c diff --git a/externals/pio1/timing/GPTLutil.c b/src/externals/pio1/timing/GPTLutil.c similarity index 100% rename from externals/pio1/timing/GPTLutil.c rename to src/externals/pio1/timing/GPTLutil.c diff --git a/externals/pio1/timing/Makefile b/src/externals/pio1/timing/Makefile similarity index 100% rename from externals/pio1/timing/Makefile rename to src/externals/pio1/timing/Makefile diff --git a/externals/pio1/timing/README b/src/externals/pio1/timing/README similarity index 100% rename from externals/pio1/timing/README rename to src/externals/pio1/timing/README diff --git a/externals/pio1/timing/XXXdotF/perf_mod.F b/src/externals/pio1/timing/XXXdotF/perf_mod.F similarity index 100% rename from externals/pio1/timing/XXXdotF/perf_mod.F rename to src/externals/pio1/timing/XXXdotF/perf_mod.F diff --git a/externals/pio1/timing/XXXdotF/perf_utils.F b/src/externals/pio1/timing/XXXdotF/perf_utils.F similarity index 100% rename from externals/pio1/timing/XXXdotF/perf_utils.F rename to src/externals/pio1/timing/XXXdotF/perf_utils.F diff --git a/externals/pio1/timing/f_wrappers.c b/src/externals/pio1/timing/f_wrappers.c similarity index 100% rename from externals/pio1/timing/f_wrappers.c rename to src/externals/pio1/timing/f_wrappers.c diff --git a/externals/pio1/timing/gptl.c b/src/externals/pio1/timing/gptl.c similarity index 100% rename from externals/pio1/timing/gptl.c rename to src/externals/pio1/timing/gptl.c diff --git a/externals/pio1/timing/gptl.h b/src/externals/pio1/timing/gptl.h similarity index 100% rename from externals/pio1/timing/gptl.h rename to src/externals/pio1/timing/gptl.h diff --git a/externals/pio1/timing/gptl.inc b/src/externals/pio1/timing/gptl.inc similarity index 100% rename from externals/pio1/timing/gptl.inc rename to src/externals/pio1/timing/gptl.inc diff --git a/externals/pio1/timing/gptl_papi.c b/src/externals/pio1/timing/gptl_papi.c similarity index 100% rename from externals/pio1/timing/gptl_papi.c rename to src/externals/pio1/timing/gptl_papi.c diff --git a/externals/pio1/timing/perf_mod.F90 b/src/externals/pio1/timing/perf_mod.F90 similarity index 100% rename from externals/pio1/timing/perf_mod.F90 rename to src/externals/pio1/timing/perf_mod.F90 diff --git a/externals/pio1/timing/perf_utils.F90 b/src/externals/pio1/timing/perf_utils.F90 similarity index 100% rename from externals/pio1/timing/perf_utils.F90 rename to src/externals/pio1/timing/perf_utils.F90 diff --git a/externals/pio1/timing/private.h b/src/externals/pio1/timing/private.h similarity index 100% rename from externals/pio1/timing/private.h rename to src/externals/pio1/timing/private.h diff --git a/externals/pio1/timing/threadutil.c b/src/externals/pio1/timing/threadutil.c similarity index 100% rename from externals/pio1/timing/threadutil.c rename to src/externals/pio1/timing/threadutil.c diff --git a/externals/pio2/.gitignore b/src/externals/pio2/.gitignore similarity index 100% rename from externals/pio2/.gitignore rename to src/externals/pio2/.gitignore diff --git a/externals/pio2/CMakeLists.txt b/src/externals/pio2/CMakeLists.txt similarity index 53% rename from externals/pio2/CMakeLists.txt rename to src/externals/pio2/CMakeLists.txt index 7ddaf9a15b8..6677ab46eb0 100644 --- a/externals/pio2/CMakeLists.txt +++ b/src/externals/pio2/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required (VERSION 2.8.12) project (PIO C Fortran) +#cmake_policy(VERSION 3.5.2) # The project version number. set(VERSION_MAJOR 2 CACHE STRING "Project major version number.") @@ -14,24 +15,43 @@ mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) #===== Library Options ===== option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON) option (PIO_ENABLE_TIMING "Enable the use of the GPTL timing library" ON) +option (PIO_ENABLE_LOGGING "Enable debug logging (large output possible)" OFF) +option (PIO_ENABLE_DOC "Enable building PIO documentation" ON) +option (PIO_ENABLE_COVERAGE "Enable code coverage" OFF) +option (PIO_ENABLE_EXAMPLES "Enable PIO examples" ON) +option (PIO_INTERNAL_DOC "Enable PIO developer documentation" OFF) option (PIO_TEST_BIG_ENDIAN "Enable test to see if machine is big endian" ON) option (PIO_USE_MPIIO "Enable support for MPI-IO auto detect" ON) option (PIO_USE_MPISERIAL "Enable mpi-serial support (instead of MPI)" OFF) option (PIO_USE_MALLOC "Use native malloc (instead of bget package)" OFF) option (WITH_PNETCDF "Require the use of PnetCDF" ON) +# Set a variable that appears in the config.h.in file. if(PIO_USE_MALLOC) set(USE_MALLOC 1) else() set(USE_MALLOC 0) endif() +# Set a variable that appears in the config.h.in file. +if(PIO_ENABLE_LOGGING) + set(ENABLE_LOGGING 1) +else() + set(ENABLE_LOGGING 0) +endif() + +if(PIO_USE_MPISERIAL) + set(USE_MPI_SERIAL 1) +else() + set(USE_MPI_SERIAL 0) +endif() + #===== Library Variables ===== set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") #===== Testing Options ===== option (PIO_ENABLE_TESTS "Enable the testing builds" ON) -option (PIO_HDF5_LOGGING "Enable hdf5 logging (requires instrumented netcdf4)" OFF) +option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) #============================================================================== # BACKWARDS COMPATIBILITY @@ -39,14 +59,14 @@ option (PIO_HDF5_LOGGING "Enable hdf5 logging (requires instrumented netcdf4)" # Old NETCDF_DIR variable --> NetCDF_PATH if (DEFINED NETCDF_DIR) - set (NetCDF_PATH ${NETCDF_DIR} - CACHE STRING "Location of the NetCDF library installation") + set (NetCDF_PATH ${NETCDF_DIR} + CACHE STRING "Location of the NetCDF library installation") endif () # Old PNETCDF_DIR variable --> PnetCDF_PATH if (DEFINED PNETCDF_DIR) - set (PnetCDF_PATH ${PNETCDF_DIR} - CACHE STRING "Location of the PnetCDF library installation") + set (PnetCDF_PATH ${PNETCDF_DIR} + CACHE STRING "Location of the PnetCDF library installation") endif () #============================================================================== @@ -58,23 +78,23 @@ list (APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) #===== External modules ===== if (NOT DEFINED USER_CMAKE_MODULE_PATH) - message (STATUS "Importing CMake_Fortran_utils") - execute_process( - COMMAND git clone https://github.com/CESM-Development/CMake_Fortran_utils - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - OUTPUT_QUIET - ERROR_QUIET) - find_path (USER_CMAKE_MODULE_PATH - NAMES mpiexec.cmake - HINTS ${CMAKE_BINARY_DIR}/CMake_Fortran_utils) - if (USER_CMAKE_MODULE_PATH) - message (STATUS "Importing CMake_Fortran_utils - success") - else () - message (FATAL_ERROR "Failed to import CMake_Fortran_utils") - endif () + message (STATUS "Importing CMake_Fortran_utils") + execute_process( + COMMAND git clone https://github.com/CESM-Development/CMake_Fortran_utils + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + OUTPUT_QUIET + ERROR_QUIET) + find_path (USER_CMAKE_MODULE_PATH + NAMES mpiexec.cmake + HINTS ${CMAKE_BINARY_DIR}/CMake_Fortran_utils) + if (USER_CMAKE_MODULE_PATH) + message (STATUS "Importing CMake_Fortran_utils - success") + else () + message (FATAL_ERROR "Failed to import CMake_Fortran_utils") + endif () endif () set (USER_CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH} - CACHE STRING "Location of the CMake_Fortran_utils") + CACHE STRING "Location of the CMake_Fortran_utils") list (APPEND CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH}) #============================================================================== @@ -84,23 +104,23 @@ list (APPEND CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH}) # System Name string (TOUPPER "${CMAKE_SYSTEM_NAME}" CMAKE_SYSTEM_NAME_CAPS) set (CMAKE_SYSTEM_DIRECTIVE "${CMAKE_SYSTEM_NAME_CAPS}" - CACHE STRING "System name preprocessor directive") + CACHE STRING "System name preprocessor directive") # C Compiler Name string (TOUPPER "${CMAKE_C_COMPILER_ID}" CMAKE_C_COMPILER_NAME) if (CMAKE_C_COMPILER_NAME STREQUAL "XL") - set (CMAKE_C_COMPILER_NAME "IBM") + set (CMAKE_C_COMPILER_NAME "IBM") endif () set (CMAKE_C_COMPILER_DIRECTIVE "CPR${CMAKE_C_COMPILER_NAME}" - CACHE STRING "C compiler name preprocessor directive") + CACHE STRING "C compiler name preprocessor directive") # Fortran Compiler Name string (TOUPPER "${CMAKE_Fortran_COMPILER_ID}" CMAKE_Fortran_COMPILER_NAME) if (CMAKE_Fortran_COMPILER_NAME STREQUAL "XL") - set (CMAKE_Fortran_COMPILER_NAME "IBM") + set (CMAKE_Fortran_COMPILER_NAME "IBM") endif () set (CMAKE_Fortran_COMPILER_DIRECTIVE "CPR${CMAKE_Fortran_COMPILER_NAME}" - CACHE STRING "Fortran compiler name preprocessor directive") + CACHE STRING "Fortran compiler name preprocessor directive") # configure a header file to pass some of the CMake settings # to the source code @@ -109,6 +129,24 @@ configure_file ( "${PROJECT_BINARY_DIR}/src/clib/config.h" ) +#============================================================================== +# SET CODE COVERAGE COMPILER FLAGS +#============================================================================== + +# Only support GNU compilers at this time +if (PIO_ENABLE_COVERAGE) + if (CMAKE_C_COMPILER_NAME STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + else () + message (WARNING "The C compiler is non-GNU: coverage of C code could NOT be enabled") + endif () + if (CMAKE_Fortran_COMPILER_NAME STREQUAL "GNU") + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fprofile-arcs -ftest-coverage") + else () + message (WARNING "The Fortran compiler is non-GNU: coverage of Fortran code could NOT be enabled") + endif () +endif () + #============================================================================== # INCLUDE SOURCE DIRECTORIES #============================================================================== @@ -116,13 +154,28 @@ configure_file ( # Libraries add_subdirectory (src) +#============================================================================== +# TESTING TARGET +#============================================================================== + +# Custom "piotests" target (builds the test executables) +add_custom_target (tests) + +# Custom "check" target that depends upon "tests" +add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}) +add_dependencies (check tests) + # Tests if (PIO_ENABLE_TESTS) - enable_testing() - include (CTest) - add_subdirectory (tests) + enable_testing() + include (CTest) + add_subdirectory (tests) + if (PIO_ENABLE_EXAMPLES) add_subdirectory (examples) + endif () endif () # Documentation -add_subdirectory (doc) +if (PIO_ENABLE_DOC) + add_subdirectory (doc) +endif () diff --git a/externals/pio2/CTestConfig.cmake b/src/externals/pio2/CTestConfig.cmake similarity index 54% rename from externals/pio2/CTestConfig.cmake rename to src/externals/pio2/CTestConfig.cmake index 3a7e8835ac9..cd7099ae6bb 100644 --- a/externals/pio2/CTestConfig.cmake +++ b/src/externals/pio2/CTestConfig.cmake @@ -10,6 +10,14 @@ set (CTEST_PROJECT_NAME "PIO") set (CTEST_NIGHTLY_START_TIME "00:00:00 EST") set (CTEST_DROP_METHOD "http") -set (CTEST_DROP_SITE "my.cdash.org") -set (CTEST_DROP_LOCATION "/submit.php?project=PIO") +if (DEFINED ENV{PIO_DASHBOARD_DROP_SITE}) + set (CTEST_DROP_SITE "$ENV{PIO_DASHBOARD_DROP_SITE}") +else () + set (CTEST_DROP_SITE "my.cdash.org") +endif () +if (DEFINED ENV{PIO_DASHBOARD_PROJECT_NAME}) + set (CTEST_DROP_LOCATION "/submit.php?project=$ENV{PIO_DASHBOARD_PROJECT_NAME}") +else () + set (CTEST_DROP_LOCATION "/submit.php?project=PIO") +endif () set (CTEST_DROP_SITE_CDASH TRUE) diff --git a/externals/pio2/CTestScript.cmake b/src/externals/pio2/CTestScript.cmake similarity index 89% rename from externals/pio2/CTestScript.cmake rename to src/externals/pio2/CTestScript.cmake index b313f1a24f5..1d0de3ccdcd 100644 --- a/externals/pio2/CTestScript.cmake +++ b/src/externals/pio2/CTestScript.cmake @@ -40,6 +40,7 @@ execute_process (COMMAND ${HOSTNAME_CMD} OUTPUT_STRIP_TRAILING_WHITESPACE) ## -- Set hostname ID (e.g., alcf, nwsc, nersc, ...) +message ("hostname is ${HOSTNAME}") # UCAR/NWSC Machines if (HOSTNAME MATCHES "^yslogin" OR @@ -47,13 +48,16 @@ if (HOSTNAME MATCHES "^yslogin" OR HOSTNAME MATCHES "^caldera" OR HOSTNAME MATCHES "^pronghorn") set (HOSTNAME_ID "nwsc") +# New UCAR/NWSC SGI Machines +elseif (HOSTNAME MATCHES "^laramie") + set (HOSTNAME_ID "nwscla") # ALCF/Argonne Machines elseif (HOSTNAME MATCHES "^mira" OR HOSTNAME MATCHES "^cetus" OR HOSTNAME MATCHES "^vesta" OR HOSTNAME MATCHES "^cooley") set (HOSTNAME_ID "alcf") -# ALCF/Argonne Machines +# NERSC Machines elseif (HOSTNAME MATCHES "^edison" OR HOSTNAME MATCHES "^cori" OR HOSTNAME MATCHES "^nid") @@ -64,6 +68,19 @@ elseif (HOSTNAME MATCHES "^h2ologin" ) # CGD local linux cluster elseif (HOSTNAME MATCHES "^hobart") set (HOSTNAME_ID "cgd") +# Argonne Linux workstations +elseif (HOSTNAME MATCHES "^compute001" OR + HOSTNAME MATCHES "^thwomp" OR + HOSTNAME MATCHES "^stomp" OR + HOSTNAME MATCHES "^crush" OR + HOSTNAME MATCHES "^crank" OR + HOSTNAME MATCHES "^steamroller" OR + HOSTNAME MATCHES "^grind" OR + HOSTNAME MATCHES "^churn" OR + HOSTNAME MATCHES "^trounce" OR + HOSTNAME MATCHES "^thrash" OR + HOSTNAME MATCHES "^vanquish") + set (HOSTNAME_ID "anlworkstation") else () if (CMAKE_SYSTEM_NAME MATCHES "Catamount") set (HOSTNAME_ID "ncsa") diff --git a/externals/pio2/README.md b/src/externals/pio2/README.md similarity index 100% rename from externals/pio2/README.md rename to src/externals/pio2/README.md diff --git a/externals/pio2/cmake/FindGPTL.cmake b/src/externals/pio2/cmake/FindGPTL.cmake similarity index 100% rename from externals/pio2/cmake/FindGPTL.cmake rename to src/externals/pio2/cmake/FindGPTL.cmake diff --git a/externals/pio2/cmake/FindHDF5.cmake b/src/externals/pio2/cmake/FindHDF5.cmake similarity index 100% rename from externals/pio2/cmake/FindHDF5.cmake rename to src/externals/pio2/cmake/FindHDF5.cmake diff --git a/externals/pio2/cmake/FindLIBRT.cmake b/src/externals/pio2/cmake/FindLIBRT.cmake similarity index 100% rename from externals/pio2/cmake/FindLIBRT.cmake rename to src/externals/pio2/cmake/FindLIBRT.cmake diff --git a/externals/pio2/cmake/FindLIBZ.cmake b/src/externals/pio2/cmake/FindLIBZ.cmake similarity index 100% rename from externals/pio2/cmake/FindLIBZ.cmake rename to src/externals/pio2/cmake/FindLIBZ.cmake diff --git a/externals/pio2/cmake/FindMPE.cmake b/src/externals/pio2/cmake/FindMPE.cmake similarity index 100% rename from externals/pio2/cmake/FindMPE.cmake rename to src/externals/pio2/cmake/FindMPE.cmake diff --git a/externals/pio2/cmake/FindMPISERIAL.cmake b/src/externals/pio2/cmake/FindMPISERIAL.cmake similarity index 100% rename from externals/pio2/cmake/FindMPISERIAL.cmake rename to src/externals/pio2/cmake/FindMPISERIAL.cmake diff --git a/externals/pio2/cmake/FindNetCDF.cmake b/src/externals/pio2/cmake/FindNetCDF.cmake similarity index 96% rename from externals/pio2/cmake/FindNetCDF.cmake rename to src/externals/pio2/cmake/FindNetCDF.cmake index 54c5cd7d6e9..344714b18ab 100644 --- a/externals/pio2/cmake/FindNetCDF.cmake +++ b/src/externals/pio2/cmake/FindNetCDF.cmake @@ -71,6 +71,11 @@ foreach (NCDFcomp IN LISTS NetCDF_FIND_VALID_COMPONENTS) DEFINITIONS -I${NetCDF_C_INCLUDE_DIR} COMMENT "whether NetCDF has parallel support") + # Check if logging enabled + set(CMAKE_REQUIRED_INCLUDES ${NetCDF_C_INCLUDE_DIR}) + set(CMAKE_REQUIRED_LIBRARIES ${NetCDF_C_LIBRARIES}) + CHECK_FUNCTION_EXISTS(nc_set_log_level NetCDF_C_LOGGING_ENABLED) + endif () # Dependencies diff --git a/externals/pio2/cmake/FindPAPI.cmake b/src/externals/pio2/cmake/FindPAPI.cmake similarity index 100% rename from externals/pio2/cmake/FindPAPI.cmake rename to src/externals/pio2/cmake/FindPAPI.cmake diff --git a/externals/pio2/cmake/FindPnetCDF.cmake b/src/externals/pio2/cmake/FindPnetCDF.cmake similarity index 100% rename from externals/pio2/cmake/FindPnetCDF.cmake rename to src/externals/pio2/cmake/FindPnetCDF.cmake diff --git a/externals/pio2/cmake/FindSZIP.cmake b/src/externals/pio2/cmake/FindSZIP.cmake similarity index 100% rename from externals/pio2/cmake/FindSZIP.cmake rename to src/externals/pio2/cmake/FindSZIP.cmake diff --git a/externals/pio2/cmake/LibCheck.cmake b/src/externals/pio2/cmake/LibCheck.cmake similarity index 95% rename from externals/pio2/cmake/LibCheck.cmake rename to src/externals/pio2/cmake/LibCheck.cmake index 6d7647e49ec..3f12bdf7965 100644 --- a/externals/pio2/cmake/LibCheck.cmake +++ b/src/externals/pio2/cmake/LibCheck.cmake @@ -1,5 +1,5 @@ include (CMakeParseArguments) - +include (CheckFunctionExists) #============================================================================== # # FUNCTIONS TO HELP WITH Check* MODULES @@ -14,17 +14,17 @@ include (CMakeParseArguments) # HINTS ... # DEFINITIONS ... # COMMENT ) -# +# function (check_macro VARIABLE) # Parse the input arguments set (oneValueArgs COMMENT NAME) set (multiValueArgs HINTS DEFINITIONS) - cmake_parse_arguments (${VARIABLE} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - + cmake_parse_arguments (${VARIABLE} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + # If the return variable is defined, already, don't continue if (NOT DEFINED ${VARIABLE}) - + message (STATUS "Checking ${${VARIABLE}_COMMENT}") find_file (${VARIABLE}_TRY_FILE NAMES ${${VARIABLE}_NAME} @@ -40,17 +40,17 @@ function (check_macro VARIABLE) else () message (STATUS "Checking ${${VARIABLE}_COMMENT} - no") endif () - + set (${VARIABLE} ${COMPILE_RESULT} CACHE BOOL "${${VARIABLE}_COMMENT}") - + else () message (STATUS "Checking ${${VARIABLE}_COMMENT} - failed") endif () - + unset (${VARIABLE}_TRY_FILE CACHE) endif () - + endfunction () #______________________________________________________________________________ @@ -60,17 +60,17 @@ endfunction () # NAME # HINTS ... # DEFINITIONS ...) -# +# function (check_version PKG) # Parse the input arguments set (oneValueArgs NAME MACRO_REGEX) set (multiValueArgs HINTS) - cmake_parse_arguments (${PKG} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments (${PKG} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # If the return variable is defined, already, don't continue if (NOT DEFINED ${PKG}_VERSION) - + message (STATUS "Checking ${PKG} version") find_file (${PKG}_VERSION_HEADER NAMES ${${PKG}_NAME} @@ -96,9 +96,9 @@ function (check_version PKG) else () message (STATUS "Checking ${PKG} version - failed") endif () - + unset (${PKG}_VERSION_HEADER CACHE) - + endif () endfunction () \ No newline at end of file diff --git a/externals/pio2/cmake/LibFind.cmake b/src/externals/pio2/cmake/LibFind.cmake similarity index 100% rename from externals/pio2/cmake/LibFind.cmake rename to src/externals/pio2/cmake/LibFind.cmake diff --git a/externals/pio2/cmake/LibMPI.cmake b/src/externals/pio2/cmake/LibMPI.cmake similarity index 75% rename from externals/pio2/cmake/LibMPI.cmake rename to src/externals/pio2/cmake/LibMPI.cmake index 9e30717e4c4..2dd9a7d27a8 100644 --- a/externals/pio2/cmake/LibMPI.cmake +++ b/src/externals/pio2/cmake/LibMPI.cmake @@ -1,5 +1,16 @@ include (CMakeParseArguments) +# Find Valgrind to perform memory leak check +if (PIO_VALGRIND_CHECK) + find_program (VALGRIND_COMMAND NAMES valgrind) + if (VALGRIND_COMMAND) + set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes) + else () + message (WARNING "Valgrind not found: memory leak check could not be performed") + set (VALGRIND_COMMAND "") + endif () +endif () + # # - Functions for parallel testing with CTest # @@ -20,6 +31,11 @@ function (platform_name RETURN_VARIABLE) SITENAME MATCHES "^caldera") set (${RETURN_VARIABLE} "nwsc" PARENT_SCOPE) + + # New NWSC SGI machine + elseif (SITENAME MATCHES "^laramie") + + set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE) # ALCF/Argonne Machines elseif (SITENAME MATCHES "^mira" OR @@ -29,7 +45,7 @@ function (platform_name RETURN_VARIABLE) set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE) - # ALCF/Argonne Machines + # NERSC Machines elseif (SITENAME MATCHES "^edison" OR SITENAME MATCHES "^cori") @@ -39,6 +55,12 @@ function (platform_name RETURN_VARIABLE) elseif (SITENAME MATCHES "^h2ologin") set (${RETURN_VARIABLE} "ncsa" PARENT_SCOPE) + + # OLCF/Oak Ridge Machines + elseif (SITENAME MATCHES "^eos" OR + SITENAME MATCHES "^titan") + + set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE) else () @@ -78,14 +100,14 @@ function (add_mpi_test TESTNAME) # Run tests directly from the command line set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} - ${MPIEXEC_PREFLAGS} ${exec_file} + ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${MPIEXEC_POSTFLAGS} ${exec_args}) else () # Run tests from the platform-specific executable set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} - ${num_procs} ${exec_file} ${exec_args}) + ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args}) endif () diff --git a/externals/pio2/cmake/TryHDF5_HAS_SZIP.c b/src/externals/pio2/cmake/TryHDF5_HAS_SZIP.c similarity index 100% rename from externals/pio2/cmake/TryHDF5_HAS_SZIP.c rename to src/externals/pio2/cmake/TryHDF5_HAS_SZIP.c diff --git a/externals/pio2/cmake/TryNetCDF_DAP.c b/src/externals/pio2/cmake/TryNetCDF_DAP.c similarity index 100% rename from externals/pio2/cmake/TryNetCDF_DAP.c rename to src/externals/pio2/cmake/TryNetCDF_DAP.c diff --git a/externals/pio2/cmake/TryNetCDF_PARALLEL.c b/src/externals/pio2/cmake/TryNetCDF_PARALLEL.c similarity index 100% rename from externals/pio2/cmake/TryNetCDF_PARALLEL.c rename to src/externals/pio2/cmake/TryNetCDF_PARALLEL.c diff --git a/externals/pio2/cmake/TryNetCDF_PNETCDF.c b/src/externals/pio2/cmake/TryNetCDF_PNETCDF.c similarity index 100% rename from externals/pio2/cmake/TryNetCDF_PNETCDF.c rename to src/externals/pio2/cmake/TryNetCDF_PNETCDF.c diff --git a/externals/pio2/cmake/mpiexec.alcf b/src/externals/pio2/cmake/mpiexec.alcf similarity index 100% rename from externals/pio2/cmake/mpiexec.alcf rename to src/externals/pio2/cmake/mpiexec.alcf diff --git a/externals/pio2/cmake/mpiexec.ncsa b/src/externals/pio2/cmake/mpiexec.ncsa similarity index 100% rename from externals/pio2/cmake/mpiexec.ncsa rename to src/externals/pio2/cmake/mpiexec.ncsa diff --git a/externals/pio2/cmake/mpiexec.nersc b/src/externals/pio2/cmake/mpiexec.nersc similarity index 100% rename from externals/pio2/cmake/mpiexec.nersc rename to src/externals/pio2/cmake/mpiexec.nersc diff --git a/externals/pio2/cmake/mpiexec.nwsc b/src/externals/pio2/cmake/mpiexec.nwsc similarity index 100% rename from externals/pio2/cmake/mpiexec.nwsc rename to src/externals/pio2/cmake/mpiexec.nwsc diff --git a/src/externals/pio2/cmake/mpiexec.nwscla b/src/externals/pio2/cmake/mpiexec.nwscla new file mode 100755 index 00000000000..bb7018bf92c --- /dev/null +++ b/src/externals/pio2/cmake/mpiexec.nwscla @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +NP=$1 +shift + +mpiexec_mpt -n $NP $@ diff --git a/src/externals/pio2/cmake/mpiexec.olcf b/src/externals/pio2/cmake/mpiexec.olcf new file mode 100755 index 00000000000..2bb0d1c8468 --- /dev/null +++ b/src/externals/pio2/cmake/mpiexec.olcf @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +NP=$1 +shift + +aprun -n $NP $@ diff --git a/externals/pio2/ctest/CTestEnvironment-alcf.cmake b/src/externals/pio2/ctest/CTestEnvironment-alcf.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-alcf.cmake rename to src/externals/pio2/ctest/CTestEnvironment-alcf.cmake diff --git a/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake new file mode 100644 index 00000000000..07ba92a2a72 --- /dev/null +++ b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake @@ -0,0 +1,25 @@ +#============================================================================== +# +# This file sets the environment variables needed to configure and build +# on Argonne Linux workstations +# +#============================================================================== + +# Assume all package locations (NetCDF, PnetCDF, HDF5, etc) are already +# set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. + +# Define the extra CMake configure options +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE") +set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DNetCDF_PATH=$ENV{NETCDFROOT}") +set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPnetCDF_PATH=$ENV{PNETCDFROOT}") +set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DHDF5_PATH=$ENV{HDF5ROOT}") + +# If ENABLE_COVERAGE environment variable is set, then enable code coverage +if (DEFINED ENV{ENABLE_COVERAGE}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_ENABLE_COVERAGE=ON") +endif () + +# If VALGRIND_CHECK environment variable is set, then enable memory leak check using Valgrind +if (DEFINED ENV{VALGRIND_CHECK}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_VALGRIND_CHECK=ON") +endif () diff --git a/externals/pio2/ctest/CTestEnvironment-cgd.cmake b/src/externals/pio2/ctest/CTestEnvironment-cgd.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-cgd.cmake rename to src/externals/pio2/ctest/CTestEnvironment-cgd.cmake diff --git a/externals/pio2/ctest/CTestEnvironment-ncsa.cmake b/src/externals/pio2/ctest/CTestEnvironment-ncsa.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-ncsa.cmake rename to src/externals/pio2/ctest/CTestEnvironment-ncsa.cmake diff --git a/externals/pio2/ctest/CTestEnvironment-nersc.cmake b/src/externals/pio2/ctest/CTestEnvironment-nersc.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-nersc.cmake rename to src/externals/pio2/ctest/CTestEnvironment-nersc.cmake diff --git a/externals/pio2/ctest/CTestEnvironment-nwsc.cmake b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake similarity index 82% rename from externals/pio2/ctest/CTestEnvironment-nwsc.cmake rename to src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake index 29884997dc8..356390f933d 100644 --- a/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +++ b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake @@ -1,7 +1,7 @@ #============================================================================== # # This file sets the environment variables needed to configure and build -# on the NCAR Wyoming Supercomputing Center systems +# on the NCAR Wyoming Supercomputing Center systems # (yellowstone/caldera/geyser). # #============================================================================== @@ -10,7 +10,7 @@ # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. # Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE") +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_DOC=OFF") # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) diff --git a/src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake b/src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake new file mode 100644 index 00000000000..b7f1d1c9aef --- /dev/null +++ b/src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake @@ -0,0 +1,18 @@ +#============================================================================== +# +# This file sets the environment variables needed to configure and build +# on the new NCAR Wyoming Supercomputing Center systems +# (laramie/cheyenne). +# +#============================================================================== + +# Assume all package locations (NetCDF, PnetCDF, HDF5, etc) are already +# set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. + +# Define the extra CMake configure options +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE") + +# If MPISERIAL environment variable is set, then enable MPISERIAL +if (DEFINED ENV{MPISERIAL}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON") +endif () diff --git a/externals/pio2/ctest/CTestEnvironment-unknown.cmake b/src/externals/pio2/ctest/CTestEnvironment-unknown.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-unknown.cmake rename to src/externals/pio2/ctest/CTestEnvironment-unknown.cmake diff --git a/externals/pio2/ctest/CTestScript-Test.cmake b/src/externals/pio2/ctest/CTestScript-Test.cmake similarity index 100% rename from externals/pio2/ctest/CTestScript-Test.cmake rename to src/externals/pio2/ctest/CTestScript-Test.cmake diff --git a/externals/pio2/ctest/runcdash-alcf-ibm.sh b/src/externals/pio2/ctest/runcdash-alcf-ibm.sh similarity index 89% rename from externals/pio2/ctest/runcdash-alcf-ibm.sh rename to src/externals/pio2/ctest/runcdash-alcf-ibm.sh index 3df9f86e333..9f36996bdaf 100755 --- a/externals/pio2/ctest/runcdash-alcf-ibm.sh +++ b/src/externals/pio2/ctest/runcdash-alcf-ibm.sh @@ -28,8 +28,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - $GIT clone https://github.com/PARALLELIO/ParallelIO src + $GIT clone https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop $CTEST -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-anlworkstation.sh b/src/externals/pio2/ctest/runcdash-anlworkstation.sh new file mode 100755 index 00000000000..44651b2f3e6 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-anlworkstation.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +source /software/common/adm/packages/softenv-1.6.2/etc/softenv-load.sh +source /software/common/adm/packages/softenv-1.6.2/etc/softenv-aliases.sh + +soft add +gcc-6.2.0 +soft add +mpich-3.2-gcc-6.2.0 +soft add +cmake-3.5.1 + +export NETCDFROOT=/soft/apps/packages/climate/netcdf/4.4.1c-4.2cxx-4.4.4f-parallel/gcc-6.2.0 +export PNETCDFROOT=/soft/apps/packages/climate/pnetcdf/1.7.0/gcc-6.2.0 +export HDF5ROOT=/soft/apps/packages/climate/hdf5/1.8.16-parallel/gcc-6.2.0 + +export CC=mpicc +export FC=mpifort + +export PIO_DASHBOARD_SITE=anlworkstation-`hostname` +export PIO_DASHBOARD_ROOT=/sandbox/dashboard +export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src +export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} +export PIO_COMPILER_ID=gcc-`gcc --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} +echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-cgd-nag.sh b/src/externals/pio2/ctest/runcdash-cgd-nag.sh similarity index 71% rename from externals/pio2/ctest/runcdash-cgd-nag.sh rename to src/externals/pio2/ctest/runcdash-cgd-nag.sh index 44ced786200..55c477b73bc 100755 --- a/externals/pio2/ctest/runcdash-cgd-nag.sh +++ b/src/externals/pio2/ctest/runcdash-cgd-nag.sh @@ -8,8 +8,8 @@ else fi module purge -module load compiler/nag/6.0 -module load tool/parallel-netcdf/1.6.1/nag/openmpi +module load compiler/nag/6.1 +module load tool/parallel-netcdf/1.7.0/nag/mvapich2 export CC=mpicc export FC=mpif90 @@ -17,7 +17,7 @@ export PIO_DASHBOARD_SITE="cgd" export PIO_DASHBOARD_ROOT=/scratch/cluster/katec/dashboard export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} -export PIO_COMPILER_ID=Nag-6.0-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` +export PIO_COMPILER_ID=Nag-6.1-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then mkdir "$PIO_DASHBOARD_ROOT" @@ -28,9 +28,11 @@ echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi - cd src +git checkout develop +git pull origin develop + ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-nersc-cray.sh b/src/externals/pio2/ctest/runcdash-nersc-cray.sh new file mode 100755 index 00000000000..d3516cea7d5 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-nersc-cray.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module rm PrgEnv-intel +module rm PrgEnv-cray +module rm PrgEnv-gnu +module rm intel +module rm cce +module rm cray-parallel-netcdf +module rm cray-parallel-hdf5 +module rm pmi +module rm cray-libsci +module rm cray-mpich2 +module rm cray-mpich +module rm cray-netcdf +module rm cray-hdf5 +module rm cray-netcdf-hdf5parallel +module rm craype-sandybridge +module rm craype-ivybridge +module rm craype-haswell +module rm craype +module load PrgEnv-cray + +case "$NERSC_HOST" in + edison) + cd $CSCRATCH/dashboard + module switch cce cce/8.5.1 + module load craype-ivybridge + module load git/2.4.6 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + cori) + cd $SCRATCH/dashboard + module switch cce cce/8.5.4 + module load craype-mic-knl + module load git/2.9.1 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + +esac + +export CC=cc +export FC=ftn + +export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_COMPILER_ID=Cray-`$CC -V 2>&1 | cut -d' ' -f5` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +export HDF5_DISABLE_VERSION_CHECK=2 + +ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-nersc-intel.sh b/src/externals/pio2/ctest/runcdash-nersc-intel.sh new file mode 100755 index 00000000000..55c80559b60 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-nersc-intel.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module rm PrgEnv-intel +module rm PrgEnv-cray +module rm PrgEnv-gnu +module rm intel +module rm cce +module rm cray-parallel-netcdf +module rm cray-parallel-hdf5 +module rm pmi +module rm cray-libsci +module rm cray-mpich2 +module rm cray-mpich +module rm cray-netcdf +module rm cray-hdf5 +module rm cray-netcdf-hdf5parallel +module rm craype-sandybridge +module rm craype-ivybridge +module rm craype-haswell +module rm craype +module load PrgEnv-intel + +case "$NERSC_HOST" in + edison) + cd $CSCRATCH/dashboard + module switch intel intel/16.0.0.109 + module load craype-ivybridge + module load git/2.4.6 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + cori) + cd $SCRATCH/dashboard + module switch intel intel/17.0.1.132 + module load craype-mic-knl + module load git/2.9.1 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + +esac + +export CC=cc +export FC=ftn + +export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +export HDF5_DISABLE_VERSION_CHECK=2 +ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-gnu.sh b/src/externals/pio2/ctest/runcdash-nwsc-gnu.sh similarity index 72% rename from externals/pio2/ctest/runcdash-nwsc-gnu.sh rename to src/externals/pio2/ctest/runcdash-nwsc-gnu.sh index 1049103dab5..d406138bcfd 100755 --- a/externals/pio2/ctest/runcdash-nwsc-gnu.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-gnu.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel gnu/5.3.0 +module swap intel gnu/6.1.0 module load git/2.3.0 module load cmake/3.0.2 -module load netcdf-mpi/4.3.3.1 -module load pnetcdf/1.6.1 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.7.0 export CC=mpicc export FC=mpif90 @@ -27,8 +27,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh similarity index 84% rename from externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh rename to src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh index 6f6ec545c23..702427e5a7b 100755 --- a/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh @@ -28,8 +28,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-intel.sh b/src/externals/pio2/ctest/runcdash-nwsc-intel.sh similarity index 71% rename from externals/pio2/ctest/runcdash-nwsc-intel.sh rename to src/externals/pio2/ctest/runcdash-nwsc-intel.sh index 53e2e85ebea..1e72e9a1b31 100755 --- a/externals/pio2/ctest/runcdash-nwsc-intel.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-intel.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel intel/16.0.0 +module swap intel intel/16.0.3 module load git/2.3.0 module load cmake/3.0.2 -module load netcdf-mpi/4.3.3.1 -module load pnetcdf/1.6.1 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.7.0 export CC=mpicc export FC=mpif90 @@ -27,8 +27,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-pgi.sh b/src/externals/pio2/ctest/runcdash-nwsc-pgi.sh similarity index 72% rename from externals/pio2/ctest/runcdash-nwsc-pgi.sh rename to src/externals/pio2/ctest/runcdash-nwsc-pgi.sh index 284e605acb3..20c09d619e3 100755 --- a/externals/pio2/ctest/runcdash-nwsc-pgi.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-pgi.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel pgi/15.10 +module swap intel pgi/16.5 module load git/2.3.0 module load cmake/3.0.2 -module load netcdf-mpi/4.3.3.1 -module load pnetcdf/1.6.1 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.7.0 export CC=mpicc export FC=mpif90 @@ -27,8 +27,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nersc-intel.sh b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh similarity index 50% rename from externals/pio2/ctest/runcdash-nersc-intel.sh rename to src/externals/pio2/ctest/runcdash-nwscla-intel.sh index bb87018b985..3812b9d61f0 100755 --- a/externals/pio2/ctest/runcdash-nersc-intel.sh +++ b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh @@ -7,20 +7,20 @@ else model=$1 fi -module purge -module load PrgEnv-intel -module load craype-ivybridge -module load cray-shmem -module load cray-mpich -module load torque -module load git/2.4.6 -module load cmake/3.0.0 -module load cray-hdf5-parallel/1.8.14 -module load cray-netcdf-hdf5parallel/4.3.3.1 -module load cray-parallel-netcdf/1.6.0 - -export CC=cc -export FC=ftn +source /etc/profile.d/modules.sh + +module reset +module unload netcdf +module swap intel intel/16.0.3 +module load git/2.10.0 +module load cmake/3.6.2 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.8.0 +echo "MODULE LIST..." +module list + +export CC=mpicc +export FC=mpif90 export PIO_DASHBOARD_ROOT=`pwd`/dashboard export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` @@ -31,8 +31,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runctest-alcf.sh b/src/externals/pio2/ctest/runctest-alcf.sh similarity index 100% rename from externals/pio2/ctest/runctest-alcf.sh rename to src/externals/pio2/ctest/runctest-alcf.sh diff --git a/src/externals/pio2/ctest/runctest-anlworkstation.sh b/src/externals/pio2/ctest/runctest-anlworkstation.sh new file mode 100755 index 00000000000..9718a834804 --- /dev/null +++ b/src/externals/pio2/ctest/runctest-anlworkstation.sh @@ -0,0 +1,18 @@ +#!/bin/sh +#============================================================================== +# +# This script defines how to run CTest on the Argonne Linux workstations. +# +# This assumes the CTest model name (e.g., "Nightly") is passed to it when +# run. +# +#============================================================================== + +# Get the CTest script directory +scrdir=$1 + +# Get the CTest model name +model=$2 + +# Run the "ctest" command in another process +ctest -S ${scrdir}/CTestScript-Test.cmake,${model} -V diff --git a/externals/pio2/ctest/runctest-cgd.sh b/src/externals/pio2/ctest/runctest-cgd.sh similarity index 86% rename from externals/pio2/ctest/runctest-cgd.sh rename to src/externals/pio2/ctest/runctest-cgd.sh index 02c99d10b4a..fccf3396d3b 100755 --- a/externals/pio2/ctest/runctest-cgd.sh +++ b/src/externals/pio2/ctest/runctest-cgd.sh @@ -1,8 +1,8 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the Argonne Leadership Computing -# Facility systems (mira/cetus/vesta/cooley). +# This script defines how to run CTest on the NCAR CGD local cluster +# Hobart. # # This assumes the CTest model name (e.g., "Nightly") is passed to it when # run. @@ -29,7 +29,7 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh` +jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh -q short` # Wait for the job to complete before exiting while true; do diff --git a/externals/pio2/ctest/runctest-ncsa.sh b/src/externals/pio2/ctest/runctest-ncsa.sh similarity index 100% rename from externals/pio2/ctest/runctest-ncsa.sh rename to src/externals/pio2/ctest/runctest-ncsa.sh diff --git a/externals/pio2/ctest/runctest-nersc.sh b/src/externals/pio2/ctest/runctest-nersc.sh similarity index 68% rename from externals/pio2/ctest/runctest-nersc.sh rename to src/externals/pio2/ctest/runctest-nersc.sh index 8c58fba0a1d..a84d26bbeb8 100755 --- a/externals/pio2/ctest/runctest-nersc.sh +++ b/src/externals/pio2/ctest/runctest-nersc.sh @@ -1,8 +1,8 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the Argonne Leadership Computing -# Facility systems (mira/cetus/vesta/cooley). +# This script defines how to run CTest on the National Energy Research +# Scientific Computing Center systems (edison/cori). # # This assumes the CTest model name (e.g., "Nightly") is passed to it when # run. @@ -19,8 +19,17 @@ model=$2 echo "#!/bin/sh" > runctest.slurm echo "#SBATCH --partition debug" >> runctest.slurm echo "#SBATCH --nodes=1" >> runctest.slurm -echo "#SBATCH --ntasks-per-node=32" >> runctest.slurm -echo "#SBATCH --time=00:15:00" >> runctest.slurm +case "$NERSC_HOST" in + edison) + echo "#SBATCH --ntasks-per-node=32" >> runctest.slurm + ;; + cori) + echo "#SBATCH --ntasks-per-node=68" >> runctest.slurm + echo "#SBATCH -C knl" >> runctest.slurm + ;; +esac + +echo "#SBATCH --time=01:00:00" >> runctest.slurm echo "#SBATCH --export PIO_DASHBOARD_SITE,PIO_DASHBOARD_BUILD_NAME,PIO_DASHBOARD_SOURCE_DIR,PIO_DASHBOARD_BINARY_DIR" >> runctest.slurm #echo "cd \$PBS_O_WORKDIR" >> runctest.pbs @@ -29,7 +38,14 @@ echo "\$CTEST_CMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.s chmod +x runctest.slurm # Submit the job to the queue #jobid=`sbatch runctest.slurm| egrep -o -e "\b[0-9]+$"` -salloc -N 1 ./runctest.slurm +case "$NERSC_HOST" in + edison) + salloc -N 1 ./runctest.slurm + ;; + cori) + salloc -N 1 -C knl ./runctest.slurm + ;; +esac # Wait for the job to complete before exiting #while true; do # status=`squeue -j $jobid` diff --git a/externals/pio2/ctest/runctest-nwsc.sh b/src/externals/pio2/ctest/runctest-nwsc.sh similarity index 100% rename from externals/pio2/ctest/runctest-nwsc.sh rename to src/externals/pio2/ctest/runctest-nwsc.sh diff --git a/src/externals/pio2/ctest/runctest-nwscla.sh b/src/externals/pio2/ctest/runctest-nwscla.sh new file mode 100755 index 00000000000..6b6f4d87f7c --- /dev/null +++ b/src/externals/pio2/ctest/runctest-nwscla.sh @@ -0,0 +1,45 @@ +#!/bin/sh +#============================================================================== +# +# This script defines how to run CTest on the NCAR Wyoming Supercomputing +# Center systems (yellowstone/caldera/geyser). +# +# This assumes the CTest model name (e.g., "Nightly") is passed to it when +# run. +# +#============================================================================== + +# Get the CTest script directory +scrdir=$1 + +# Get the CTest model name +model=$2 + +# Write QSUB submission script with the test execution command +echo "#!/bin/sh" > runctest.sh +echo "#PBS -l walltime=01:00:00" >> runctest.sh +echo "#PBS -l select=1:ncpus=8:mpiprocs=8" >> runctest.sh +echo "#PBS -A SCSG0002" >> runctest.sh +echo "export PIO_DASHBOARD_SITE=nwscla-${HOSTNAME}" >> runctest.sh +echo "CTESTCMD=`which ctest`" >> runctest.sh +echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh + +# Make the QSUB script executable +chmod +x runctest.sh + +# Submit the job to the queue +jobid=`qsub runctest.sh` + +# Wait for the job to complete before exiting +while true; do + status=`qstat $jobid` + echo $status + if [ "$status" == "" ]; then + break + else + sleep 10 + fi +done + +exit 0 + diff --git a/externals/pio2/ctest/runctest-unknown.sh b/src/externals/pio2/ctest/runctest-unknown.sh similarity index 100% rename from externals/pio2/ctest/runctest-unknown.sh rename to src/externals/pio2/ctest/runctest-unknown.sh diff --git a/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake b/src/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake similarity index 100% rename from externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake rename to src/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake diff --git a/externals/pio2/doc/CMakeFiles/CMakeOutput.log b/src/externals/pio2/doc/CMakeFiles/CMakeOutput.log similarity index 100% rename from externals/pio2/doc/CMakeFiles/CMakeOutput.log rename to src/externals/pio2/doc/CMakeFiles/CMakeOutput.log diff --git a/externals/pio2/doc/CMakeFiles/cmake.check_cache b/src/externals/pio2/doc/CMakeFiles/cmake.check_cache similarity index 100% rename from externals/pio2/doc/CMakeFiles/cmake.check_cache rename to src/externals/pio2/doc/CMakeFiles/cmake.check_cache diff --git a/src/externals/pio2/doc/CMakeLists.txt b/src/externals/pio2/doc/CMakeLists.txt new file mode 100644 index 00000000000..982b445b66a --- /dev/null +++ b/src/externals/pio2/doc/CMakeLists.txt @@ -0,0 +1,43 @@ +#============================================================================== +# +# API documentation with Doxygen +# +#============================================================================== + +find_package(Doxygen) + +if(DOXYGEN_FOUND) + # This supports the build with/witout internal documentation. + if (PIO_INTERNAL_DOC) + SET(C_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/../src/clib") + else () + SET(C_SRC_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pioc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_nc4.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_darray.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_get_nc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_put_nc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_varm.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_file.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio.h \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_nc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/topology.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pioc_sc.c" ) + endif () + + # Process the Doxyfile using options set during configure. + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in + ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) + + # Copy necessary files. + add_custom_target(doc + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" VERBATIM) +endif(DOXYGEN_FOUND) diff --git a/externals/pio2/doc/Doxyfile.in b/src/externals/pio2/doc/Doxyfile.in similarity index 99% rename from externals/pio2/doc/Doxyfile.in rename to src/externals/pio2/doc/Doxyfile.in index 31dc3ffd113..d9fe2ca3260 100644 --- a/externals/pio2/doc/Doxyfile.in +++ b/src/externals/pio2/doc/Doxyfile.in @@ -769,11 +769,14 @@ WARN_LOGFILE = # Note: If this tag is empty the current directory is searched. INPUT = @CMAKE_CURRENT_SOURCE_DIR@/source \ - @CMAKE_CURRENT_SOURCE_DIR@/../src/clib \ @CMAKE_CURRENT_SOURCE_DIR@/../src/flib \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \ - @CMAKE_BINARY_DIR@/src/flib + @CMAKE_BINARY_DIR@/src/flib \ + @C_SRC_FILES@ + +# Uncomment this after the async code is fully merged into PIO. +# @CMAKE_CURRENT_SOURCE_DIR@/../src/clib # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/externals/pio2/doc/DoxygenLayout.xml b/src/externals/pio2/doc/DoxygenLayout.xml similarity index 100% rename from externals/pio2/doc/DoxygenLayout.xml rename to src/externals/pio2/doc/DoxygenLayout.xml diff --git a/externals/pio2/doc/customdoxygen.css b/src/externals/pio2/doc/customdoxygen.css similarity index 100% rename from externals/pio2/doc/customdoxygen.css rename to src/externals/pio2/doc/customdoxygen.css diff --git a/externals/pio2/doc/doxygen.sty b/src/externals/pio2/doc/doxygen.sty similarity index 100% rename from externals/pio2/doc/doxygen.sty rename to src/externals/pio2/doc/doxygen.sty diff --git a/externals/pio2/doc/images/baseimage.graffle b/src/externals/pio2/doc/images/baseimage.graffle similarity index 100% rename from externals/pio2/doc/images/baseimage.graffle rename to src/externals/pio2/doc/images/baseimage.graffle diff --git a/externals/pio2/doc/images/block-cyclic-rearr.eps b/src/externals/pio2/doc/images/block-cyclic-rearr.eps similarity index 100% rename from externals/pio2/doc/images/block-cyclic-rearr.eps rename to src/externals/pio2/doc/images/block-cyclic-rearr.eps diff --git a/externals/pio2/doc/images/block-cyclic-rearr.graffle b/src/externals/pio2/doc/images/block-cyclic-rearr.graffle similarity index 100% rename from externals/pio2/doc/images/block-cyclic-rearr.graffle rename to src/externals/pio2/doc/images/block-cyclic-rearr.graffle diff --git a/externals/pio2/doc/images/block-cyclic-rearr.png b/src/externals/pio2/doc/images/block-cyclic-rearr.png similarity index 100% rename from externals/pio2/doc/images/block-cyclic-rearr.png rename to src/externals/pio2/doc/images/block-cyclic-rearr.png diff --git a/externals/pio2/doc/images/block-cyclic.eps b/src/externals/pio2/doc/images/block-cyclic.eps similarity index 100% rename from externals/pio2/doc/images/block-cyclic.eps rename to src/externals/pio2/doc/images/block-cyclic.eps diff --git a/externals/pio2/doc/images/block-cyclic.graffle b/src/externals/pio2/doc/images/block-cyclic.graffle similarity index 100% rename from externals/pio2/doc/images/block-cyclic.graffle rename to src/externals/pio2/doc/images/block-cyclic.graffle diff --git a/externals/pio2/doc/images/block-cyclic.png b/src/externals/pio2/doc/images/block-cyclic.png similarity index 100% rename from externals/pio2/doc/images/block-cyclic.png rename to src/externals/pio2/doc/images/block-cyclic.png diff --git a/externals/pio2/doc/images/dof-rearr.eps b/src/externals/pio2/doc/images/dof-rearr.eps similarity index 100% rename from externals/pio2/doc/images/dof-rearr.eps rename to src/externals/pio2/doc/images/dof-rearr.eps diff --git a/externals/pio2/doc/images/dof-rearr.graffle b/src/externals/pio2/doc/images/dof-rearr.graffle similarity index 100% rename from externals/pio2/doc/images/dof-rearr.graffle rename to src/externals/pio2/doc/images/dof-rearr.graffle diff --git a/externals/pio2/doc/images/dof-rearr.png b/src/externals/pio2/doc/images/dof-rearr.png similarity index 100% rename from externals/pio2/doc/images/dof-rearr.png rename to src/externals/pio2/doc/images/dof-rearr.png diff --git a/externals/pio2/doc/images/dof.eps b/src/externals/pio2/doc/images/dof.eps similarity index 100% rename from externals/pio2/doc/images/dof.eps rename to src/externals/pio2/doc/images/dof.eps diff --git a/externals/pio2/doc/images/dof.graffle b/src/externals/pio2/doc/images/dof.graffle similarity index 100% rename from externals/pio2/doc/images/dof.graffle rename to src/externals/pio2/doc/images/dof.graffle diff --git a/externals/pio2/doc/images/dof.png b/src/externals/pio2/doc/images/dof.png similarity index 100% rename from externals/pio2/doc/images/dof.png rename to src/externals/pio2/doc/images/dof.png diff --git a/externals/pio2/doc/source/CAMexample.txt b/src/externals/pio2/doc/source/CAMexample.txt similarity index 100% rename from externals/pio2/doc/source/CAMexample.txt rename to src/externals/pio2/doc/source/CAMexample.txt diff --git a/externals/pio2/doc/source/Decomp.txt b/src/externals/pio2/doc/source/Decomp.txt similarity index 100% rename from externals/pio2/doc/source/Decomp.txt rename to src/externals/pio2/doc/source/Decomp.txt diff --git a/externals/pio2/doc/source/Error.txt b/src/externals/pio2/doc/source/Error.txt similarity index 88% rename from externals/pio2/doc/source/Error.txt rename to src/externals/pio2/doc/source/Error.txt index 4664919cbc8..72c0da23e20 100644 --- a/externals/pio2/doc/source/Error.txt +++ b/src/externals/pio2/doc/source/Error.txt @@ -21,9 +21,6 @@ describing the error and then calling mpi_abort. Application developers can change this behaivior with a call to \ref PIO_seterrorhandling -For example, if a developer wanted -to see if an input netcdf file contained the variable 'U' they might do the following: - \verbinclude errorhandle \copydoc PIO_error_method diff --git a/externals/pio2/doc/source/Examples.txt b/src/externals/pio2/doc/source/Examples.txt similarity index 99% rename from externals/pio2/doc/source/Examples.txt rename to src/externals/pio2/doc/source/Examples.txt index f2af4ebcf66..e18d2926c7c 100644 --- a/externals/pio2/doc/source/Examples.txt +++ b/src/externals/pio2/doc/source/Examples.txt @@ -41,6 +41,8 @@ The Fortran examples are in the examples/f03 subdirectory. - examplePio.f90 A simple example showing a write, then read, of a 1D variable. +### Other Examples + PIO has been implemented in several geophysical component models, including the Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program (POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7). We also provide diff --git a/externals/pio2/doc/source/GettingStarted.txt b/src/externals/pio2/doc/source/GettingStarted.txt similarity index 100% rename from externals/pio2/doc/source/GettingStarted.txt rename to src/externals/pio2/doc/source/GettingStarted.txt diff --git a/externals/pio2/doc/source/Installing.txt b/src/externals/pio2/doc/source/Installing.txt similarity index 99% rename from externals/pio2/doc/source/Installing.txt rename to src/externals/pio2/doc/source/Installing.txt index 0c0020d39ac..6f281152248 100644 --- a/externals/pio2/doc/source/Installing.txt +++ b/src/externals/pio2/doc/source/Installing.txt @@ -20,7 +20,7 @@ The PIO code is currently stored on github at setenv DAV_CORES 4 + > execca ctest + +## PIO2 Performance Test + +To run the performance tests, you will need to add two files to the **tests/performance** subdirectory of the PIO build directory. First, you will need a decomp file. You can download one from our google code page here: +https://svn-ccsm-piodecomps.cgd.ucar.edu/trunk/ . +You can use any of these files, and save them to your home or base work directory. Secondly, you will need to add a namelist file, named "pioperf.nl". Save this file in the directory with your **pioperf** executable (this is found in the **tests/performance** subdirectory of the PIO build directory). + + +The contents of the namelist file should look like: + + &pioperf + + decompfile = "/u/home/user/piodecomp30tasks01dims06.dat" + + pio_typenames = 'pnetcdf' + + niotasks = 30 + + rearrangers = 1 + + nvars = 2 + + / + +Here, the second line ("decompfile") points to the path for your decomp file (wherever you saved it). For the rest of the lines, each item added to the list adds another test to be run. For instance, to test all of the types of supported IO, your pio_typenames would look like: + + pio_typenames = 'pnetcdf','netcdf','netcdf4p','netcdf4c' + +HDF5 is netcdf4p, and Parallel-Netcdf is pnetcdf. + +To test with different numbers of IO tasks, you could do: + + niotasks = 30,15,5 + +(These tasks are the subset of the run tasks that are designated IO tasks) + +To test with both of the rearranger algorithms: + + rearrangers = 1,2 + +(Each rearranger is a different algorithm for converting from data in memory to data in a file on disk. The first one, BOX, is the older method from PIO1, the second, SUBSET, is a newer method that seems to be more efficient in large numbers of tasks) + +To test with different numbers of variables: + + nvars = 8,5,3,2 + +(The more variables you use, the higher data throughput goes, usually) + +To run, submit a job with 'pioperf' as the executable, and at least as many tasks as you have specified in the decomposition file. On yellowstone, a submit script could look like: + + #!/bin/tcsh + + #BSUB -P P00000000 # project code + #BSUB -W 00:10 # wall-clock time (hrs:mins) + #BSUB -n 30 # number of tasks in job + #BSUB -R "span[ptile=16]" # run 16 MPI tasks per node + #BSUB -J pio_perftest # job name + #BSUB -o pio_perftest.%J.out # output file name in which %J is replaced by the job ID + #BSUB -e pio_perftest.%J.err # error file name in which %J is replaced by the job ID + #BSUB -q small # queue + + #run the executable + mpirun.lsf /glade/p/work/katec/pio_work/pio_build/tests/performance/pioperf + +The result(s) will look like a line in the output file such as: +~~~~~~~~~~~~~~ +RESULT: write BOX 4 30 2 16.9905924688 +~~~~~~~~~~~~~~ + +You can decode this as: +1. Read/write describes the io operation performed +2. BOX/SUBSET is the algorithm for the rearranger (as described above) +3. 4 [1-4] is the io library used for the operation. The options here are [1] Parallel-netcdf [2] NetCDF3 [3] NetCDF4-Compressed [4] NetCDF4-Parallel +4. 30 [any number] is the number of io-specific tasks used in the operation. Must be less than the number of MPI tasks used in the test. +5. 2 [any number] is the number of variables read or written during the operation +6. 16.9905924688 [any number] is the Data Rate of the operation in MB/s. This is the important value for determining performance of the system. The higher this numbre is, the better the PIO2 library is performing for the given operation. + +_Last updated: 05-17-2016_ +*/ diff --git a/externals/pio2/doc/source/api.txt b/src/externals/pio2/doc/source/api.txt similarity index 100% rename from externals/pio2/doc/source/api.txt rename to src/externals/pio2/doc/source/api.txt diff --git a/externals/pio2/doc/source/base.txt b/src/externals/pio2/doc/source/base.txt similarity index 96% rename from externals/pio2/doc/source/base.txt rename to src/externals/pio2/doc/source/base.txt index 84079df79eb..9a0401076a9 100644 --- a/externals/pio2/doc/source/base.txt +++ b/src/externals/pio2/doc/source/base.txt @@ -40,9 +40,12 @@ Please review the ChangeLog that is included with the distribution for up-to-dat - \ref intro - \ref install + - \ref mach_walkthrough - \ref decomp - \ref error + - \ref test - \ref examp - \ref faq - \ref api + - \ref contributing_code */ diff --git a/src/externals/pio2/doc/source/contributing_code.txt b/src/externals/pio2/doc/source/contributing_code.txt new file mode 100644 index 00000000000..6dbd7089826 --- /dev/null +++ b/src/externals/pio2/doc/source/contributing_code.txt @@ -0,0 +1,268 @@ +/****************************************************************************** + * + * + * + * Copyright (C) 2016 + * + * Permission to use, copy, modify, and distribute this software and its + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software + * for any purpose. It is provided "as is" without express or implied warranty. + * See the GNU General Public License for more details. + * + * Documents produced by Doxygen are derivative works derived from the + * input used in their production; they are not affected by this license. + * + */ /*! \page code_style Code Style for Contributors + +# Introduction # + +This document describes the code style to be used when contributing to +the PIO library. + +Consistency of style allows bugs to be spotted more quickly and +contributes to code quality. + +# C Code # + +## Warnings ## + +

The C library compiles under GNU gcc without warnings. No code will +be merged with the C library which causes warnings during compile. + +## Backward Compatibility ## + +

We cannot remove or change a function in the public C API. We can +add to it, carefully. + +## Brevity ## + +

Consider: C is a terse language. + +

    +
  • Use the fact that NULL == 0, and 0 == false, to shorten code. +
  • Assign and check return values in the same if statement. +
+ +## Indentation and Spacing## + +
    +
  • Use spaces not tabs. +
  • 4 spaces is the unit of intendation. +
  • Indentation as defined by the "linux" style in emacs (see below). +
  • Use spaces around most operators (=+-*/) not pointer or prefix/postfile (*++--) +
  • Spaces after most keywords (if, for, while, etc.) +
  • No spaces after function name. +
+ +## Braces ## + +

Put braces on their own line, avoiding their use if possible. + +## Documentation ## + +

    +
  • Every function must be documented using doxygen. +
  • Keep internal functions in separate code files, so that Doxygen +can easily build user and development builds of the documentation. +
  • Use the doxygen \@ingroup to put public functions in the correct +group. +
  • Code must be reasonably documented as to intention. +
  • Documentation quality and quantity are part of code review +process. +
  • Document in complete sentences. +
  • Use C (not C++) comment delimiters. +
+ +## Emacs ## + +

Put this in your .emacs file: + +

+(setq c-default-style "linux"
+          c-basic-offset 4)
+
+ +

The tab key (used anywhere on the line) will indent a line +correctly. M-x indent-region will indent a selected region of code. + +

M-x untabify will convert all the tabs in a file to spaces. + +## Code Review ## + +

    + +
  • All code is subject to review. + +
  • Pull requests will be focused on one issue. + +
  • Pull requests may not be submitted until all tests pass. + +
  • All non-trivial pull requests are associated with a github +issue. The issue is where discussion of requirements and +implementation details can be worked out. + +
  • Pull requests will be left up on github for about a day. Request +more time if you need it and are actively reviewing the code. (Note +that pull request can also be reviewed after they are merged, if you +miss one). + +
  • Jim will identify key stakeholders in changed code and ensure they +accept code changes. + +
  • Reviewers are open-minded and ready to accept improvements to the +library. + +
  • Reviewers will make comments on the pull request. All comments +must be resolved. + +
  • If chages are dictated, they happen on the branch, so code +reviewers can see the updated code. + +
  • The pull request is only merged when all programmers agree that +all issues have been resolved. + +
+ +## Merge Proceedure + +
    + +
  • Programmers begin work on a feature or fix by branching from +develop. + +
  • When a branch is ready, it is submitted to code review. + +
  • When code review is complete, and the changes are approved, the PR +is merged into the develop branch. + +
  • Mutliple merges into the develop branch may take place between +test cycles. (???) + +
  • The develop branch is tested automatically by Jenkins. + +
  • The develop branch is tested periodically by CDash (every ~6 +hours). + +
  • After all jenkins and Cdash builds complete successfully, with all +tests passing, and no warnings, the PR is merged into master by the +integrator. + +
  • Multiple PRs may be merged to master between test cycles. (???) + +
  • The branch is then deleted by whomever merged it to master. + +
  • The master branch is then tested on Jenkins. + +
  • The master branch is tested on CDash. Any test failures and the +merge to master will be rolled back. + +
+ +## Formatting Example ## + +
+/** 
+ * \@ingroup PIOc_inq_attname
+ * The PIO-C interface for the NetCDF function nc_inq_attname.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * \@param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * \@param varid the variable ID.
+ * \@param attnum the attribute ID.
+ * \@return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) 
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid,
+         attnum));
+
+    /* Find the info about this file. */
+    if (!(file = pio_get_file_from_id(ncid)))
+        return PIO_EBADID;
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_ATTNAME;
+            char name_present = name ? true : false;
+
+            if(ios->compmaster) 
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+            
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);
+#endif /* _PNETCDF */
+#ifdef _NETCDF
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_attname(file->fh, varid, attnum, name);
+#endif /* _NETCDF */
+        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+    {
+        check_mpi(file, mpierr, __FILE__, __LINE__);            
+        return PIO_EIO;
+    }
+    check_netcdf(file, ierr, __FILE__, __LINE__);
+    
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (!ierr)
+        if (name)
+        {
+            int namelen = strlen(name);
+            if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+                check_mpi(file, mpierr, __FILE__, __LINE__);
+            if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot,
+                                    ios->my_comm)))
+                check_mpi(file, mpierr, __FILE__, __LINE__);
+        }
+
+    return ierr;
+}
+
+ +## Further Information ## + +

For style issues not already covered in this document, see this style +guide. + +_Last updated: 05-16-2016_ +*/ diff --git a/externals/pio2/doc/source/example/errorhandle b/src/externals/pio2/doc/source/example/errorhandle similarity index 100% rename from externals/pio2/doc/source/example/errorhandle rename to src/externals/pio2/doc/source/example/errorhandle diff --git a/externals/pio2/doc/source/example/simple-bc b/src/externals/pio2/doc/source/example/simple-bc similarity index 100% rename from externals/pio2/doc/source/example/simple-bc rename to src/externals/pio2/doc/source/example/simple-bc diff --git a/externals/pio2/doc/source/example/simple-bc-rearr b/src/externals/pio2/doc/source/example/simple-bc-rearr similarity index 100% rename from externals/pio2/doc/source/example/simple-bc-rearr rename to src/externals/pio2/doc/source/example/simple-bc-rearr diff --git a/externals/pio2/doc/source/example/simple-bc-rearr-pe1 b/src/externals/pio2/doc/source/example/simple-bc-rearr-pe1 similarity index 100% rename from externals/pio2/doc/source/example/simple-bc-rearr-pe1 rename to src/externals/pio2/doc/source/example/simple-bc-rearr-pe1 diff --git a/externals/pio2/doc/source/example/simple-bc-rearr-pe2 b/src/externals/pio2/doc/source/example/simple-bc-rearr-pe2 similarity index 100% rename from externals/pio2/doc/source/example/simple-bc-rearr-pe2 rename to src/externals/pio2/doc/source/example/simple-bc-rearr-pe2 diff --git a/externals/pio2/doc/source/example/simple-dof b/src/externals/pio2/doc/source/example/simple-dof similarity index 100% rename from externals/pio2/doc/source/example/simple-dof rename to src/externals/pio2/doc/source/example/simple-dof diff --git a/externals/pio2/doc/source/example/simple-dof-rearr b/src/externals/pio2/doc/source/example/simple-dof-rearr similarity index 100% rename from externals/pio2/doc/source/example/simple-dof-rearr rename to src/externals/pio2/doc/source/example/simple-dof-rearr diff --git a/externals/pio2/doc/source/faq.txt b/src/externals/pio2/doc/source/faq.txt similarity index 100% rename from externals/pio2/doc/source/faq.txt rename to src/externals/pio2/doc/source/faq.txt diff --git a/externals/pio2/doc/source/mach_walkthrough.txt b/src/externals/pio2/doc/source/mach_walkthrough.txt similarity index 87% rename from externals/pio2/doc/source/mach_walkthrough.txt rename to src/externals/pio2/doc/source/mach_walkthrough.txt index 7741c44e8e4..820a6a3f288 100644 --- a/externals/pio2/doc/source/mach_walkthrough.txt +++ b/src/externals/pio2/doc/source/mach_walkthrough.txt @@ -44,7 +44,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
+ GNU @@ -54,7 +54,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
+ PGI @@ -64,7 +64,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1

  • Environment Variables @@ -103,7 +103,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + GNU @@ -117,7 +117,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + Cray @@ -132,7 +132,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
  • Environment Variables @@ -178,7 +178,7 @@ And then set the following environment variables to add in the rest of the libra %> setenv LIBZ /soft/libraries/alcf/current/xl/ZLIB
    %> setenv HDF5 /soft/libraries/hdf5/1.8.14/cnk-xl/V1R2M2-20150213
    %> setenv NETCDF /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/V1R2M2-20150213
    - %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.0/cnk-xl/V1R2M2-20150213
    + %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.1/cnk-xl/V1R2M2-20150213
    %> setenv CC /soft/compilers/wrappers/xl/mpixlc_r
    %> setenv FC /soft/compilers/wrappers/xl/mpixlf90_r
    @@ -209,7 +209,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + PGI @@ -219,7 +219,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
  • Environment Variables @@ -311,6 +311,13 @@ downloads page. (These instructions were tested using version

    Now you should be able to access mpicc, mpifort, and mpirun from the command line. (If not, make sure /usr/local/bin is in path.) +

    Note that if you wish to use valgrind on your programs, you should +configure MPICH like this: +

    CPPFLAGS=-I/usr/include/valgrind ./configure --prefix=/usr/local --enable-g=mem,meminit
    + +

    For this to work you must have packages valgrind and valgrind-devel +installed. +

  • Installing Zlib

    Download from the zlib @@ -343,26 +350,31 @@ sudo make install

    Download parallel-netcdf from the parallel-netcdf download -page. (These instructions were tested using version 1.6.1). Untar -with: 

    tar zxf parallel-netcdf-1.6.1.tar.gz
    +page. (These instructions were tested using version 1.7.0). Untar +with: 
    tar zxf parallel-netcdf-1.7.0.tar.gz

    Build with: -

    cd parallel-netcdf-1.6.1
-FC=mpifort CC=mpicc CFLAGS=-fPIC ./configure --prefix=/usr/local
+
    cd parallel-netcdf-1.7.0
+FC=mpifort CC=mpicc CFLAGS=-fPIC ./configure --prefix=/usr/local/pnetcdf-1.7.0
 make all check
 sudo make install 
    
 
+(This was also tested with version 1.6.1.)
+
 
  • Installing HDF5
 
 
    Download HDF5 from the HDF5 download
-page. (These instructions were tested using version 1.8.15-patch1). Untar
-with: 
    tar zxf hdf5-1.8.15-patch1.tar.gz
    
+page.
 
+
    1.10.0-patch1
    
+
+
    These instructions were tested using version 1.10.0-patch1. Untar
+with: 
    tar zxf hdf5-1.10.0-patch1.tar.gz
    
 
 
    Build with:
-
    cd hdf5-1.8.15-patch1
+
    cd hdf5-1.10.0-patch1
 CC=mpicc ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --enable-parallel
 make all check
 sudo make install 
    
@@ -376,8 +388,8 @@ with: 
    tar zxf netcdf-c-4.4.0.tar.gz
    
 
 
 
    Build with:
-
    cd netcdf-c-4.4.0
-CC=mpicc ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --with-hdf5=/usr/local --enable-pnetcdf --enable-parallel-tests --enable-logging
+
    cd netcdf-c-4.4.1
+CC=mpicc ./configure --enable-parallel-tests --prefix=/usr/local/netcdf-c-4.4.1 --with-hdf5=/usr/local --with-zlib=/usr/local --with-szlib=/usr/local
 make all check
 sudo make install 
    
 
@@ -391,8 +403,7 @@ with: 
    tar zxf netcdf-fortran-4.4.3.tar.gz
    
 
 
    Build with:
 
    cd netcdf-fortran-4.4.3
-F77=mpif77 F90=mpif90  ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --with-hdf5=/usr/local --enable-pnetcdf --enable-parallel-tests --enable-logging
-make all check
+make distclean; LD_LIBRARY_PATH=/usr/local/netcdf-c-4.4.1/lib FC=mpifort CPPFLAGS=-I/usr/local/netcdf-c-4.4.1/include LDFLAGS=-L/usr/local/netcdf-c-4.4.1/lib CC=mpicc ./configure --enable-parallel-tests --prefix=/usr/local/netcdf-c-4.4.1 && make check
 sudo make install 
    
 
 
  • Installing ParallelIO Library
@@ -415,12 +426,14 @@ build from the command line, including tests:
 
 
     cd build
-rm -rf * && CFLAGS=-g FFLAGS=-g CC=mpicc FC=mpifort cmake -DNetCDF_C_PATH=/usr/local -DNetCDF_Fortran_PATH=/usr/local -DPnetCDF_PATH=/usr/local -DPIO_HDF5_LOGGING=On .. && make VERBOSE=1 all tests check
+rm -rf * && CFLAGS='-Wall -g' FFLAGS=-g CC=mpicc FC=mpifort cmake -DNetCDF_C_PATH=/usr/local/netcdf-4.4.1 -DNetCDF_Fortran_PATH=/usr/local/netcdf-fortran-4.4.4 -DPnetCDF_PATH=/usr/local/pnetcdf-1.8.1 -DPIO_ENABLE_LOGGING=On .. && make VERBOSE=1 all tests check
 
    
 
-
    (Note the optional CFLAGS=-g which allows the use of a debugger
+
    Note the optional CFLAGS=-g which allows the use of a debugger
 with this code. Also note the optional VERBOSE=1 provided to make,
-which causes the build commands to be made visible.)
+which causes the build commands to be made visible.
+
+
    Note also the -DPIO_ENABLE_LOGGING=On which is helpful for debugging but should probably not be used in production builds.
 
 
  • Building and Running Performance Tests
 
@@ -477,5 +490,5 @@ variables to these tests will increase the time significantly.
 
 
 
-_Last updated: 11-18-2015_
+_Last updated: 05-16-2016_
 */
diff --git a/externals/pio2/doc/source/testpio_example.txt b/src/externals/pio2/doc/source/testpio_example.txt
similarity index 100%
rename from externals/pio2/doc/source/testpio_example.txt
rename to src/externals/pio2/doc/source/testpio_example.txt
diff --git a/externals/pio2/examples/CMakeLists.txt b/src/externals/pio2/examples/CMakeLists.txt
similarity index 64%
rename from externals/pio2/examples/CMakeLists.txt
rename to src/externals/pio2/examples/CMakeLists.txt
index ec8f36bded1..cd4fc6c23df 100644
--- a/externals/pio2/examples/CMakeLists.txt
+++ b/src/externals/pio2/examples/CMakeLists.txt
@@ -2,13 +2,17 @@
 ### CMakeList.txt for examples using pio
 ###-------------------------------------------------------------------------###
 
-if ("${PLATFORM}" STREQUAL "goldbach" )
+if (PIO_ENABLE_FORTRAN)
+  if ("${PLATFORM}" STREQUAL "goldbach" )
     ADD_SUBDIRECTORY(f03)
-elseif ("${PLATFORM}" STREQUAL "yellowstone" )
+  elseif ("${PLATFORM}" STREQUAL "yellowstone" )
     ADD_SUBDIRECTORY(f03)
     ADD_SUBDIRECTORY(c)
-else()
+  else()
     ADD_SUBDIRECTORY(f03)
     ADD_SUBDIRECTORY(c)
-#    ADD_SUBDIRECTORY(cxx)
+    #    ADD_SUBDIRECTORY(cxx)
+  endif()
+else()
+  ADD_SUBDIRECTORY(c)
 endif()
diff --git a/externals/pio2/examples/basic/CAM05.csh b/src/externals/pio2/examples/basic/CAM05.csh
similarity index 100%
rename from externals/pio2/examples/basic/CAM05.csh
rename to src/externals/pio2/examples/basic/CAM05.csh
diff --git a/externals/pio2/examples/basic/CMakeLists.txt b/src/externals/pio2/examples/basic/CMakeLists.txt
similarity index 100%
rename from externals/pio2/examples/basic/CMakeLists.txt
rename to src/externals/pio2/examples/basic/CMakeLists.txt
diff --git a/externals/pio2/examples/basic/MPASA30km.csh b/src/externals/pio2/examples/basic/MPASA30km.csh
similarity index 100%
rename from externals/pio2/examples/basic/MPASA30km.csh
rename to src/externals/pio2/examples/basic/MPASA30km.csh
diff --git a/externals/pio2/examples/basic/MPASA60km.csh b/src/externals/pio2/examples/basic/MPASA60km.csh
similarity index 100%
rename from externals/pio2/examples/basic/MPASA60km.csh
rename to src/externals/pio2/examples/basic/MPASA60km.csh
diff --git a/externals/pio2/examples/basic/POPB.csh b/src/externals/pio2/examples/basic/POPB.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPB.csh
rename to src/externals/pio2/examples/basic/POPB.csh
diff --git a/externals/pio2/examples/basic/POPC.csh b/src/externals/pio2/examples/basic/POPC.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPC.csh
rename to src/externals/pio2/examples/basic/POPC.csh
diff --git a/externals/pio2/examples/basic/POPD.csh b/src/externals/pio2/examples/basic/POPD.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPD.csh
rename to src/externals/pio2/examples/basic/POPD.csh
diff --git a/externals/pio2/examples/basic/POPDv0.csh b/src/externals/pio2/examples/basic/POPDv0.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv0.csh
rename to src/externals/pio2/examples/basic/POPDv0.csh
diff --git a/externals/pio2/examples/basic/POPDv1.csh b/src/externals/pio2/examples/basic/POPDv1.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv1.csh
rename to src/externals/pio2/examples/basic/POPDv1.csh
diff --git a/externals/pio2/examples/basic/POPDv2.csh b/src/externals/pio2/examples/basic/POPDv2.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv2.csh
rename to src/externals/pio2/examples/basic/POPDv2.csh
diff --git a/externals/pio2/examples/basic/POPDv3.csh b/src/externals/pio2/examples/basic/POPDv3.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv3.csh
rename to src/externals/pio2/examples/basic/POPDv3.csh
diff --git a/externals/pio2/examples/basic/POPDv4.csh b/src/externals/pio2/examples/basic/POPDv4.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv4.csh
rename to src/externals/pio2/examples/basic/POPDv4.csh
diff --git a/externals/pio2/examples/basic/POPDv5.csh b/src/externals/pio2/examples/basic/POPDv5.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv5.csh
rename to src/externals/pio2/examples/basic/POPDv5.csh
diff --git a/externals/pio2/examples/basic/README.testpio b/src/externals/pio2/examples/basic/README.testpio
similarity index 100%
rename from externals/pio2/examples/basic/README.testpio
rename to src/externals/pio2/examples/basic/README.testpio
diff --git a/externals/pio2/examples/basic/WRFB.csh b/src/externals/pio2/examples/basic/WRFB.csh
similarity index 100%
rename from externals/pio2/examples/basic/WRFB.csh
rename to src/externals/pio2/examples/basic/WRFB.csh
diff --git a/externals/pio2/examples/basic/alloc_mod.F90.in b/src/externals/pio2/examples/basic/alloc_mod.F90.in
similarity index 100%
rename from externals/pio2/examples/basic/alloc_mod.F90.in
rename to src/externals/pio2/examples/basic/alloc_mod.F90.in
diff --git a/externals/pio2/examples/basic/build_defaults.xml b/src/externals/pio2/examples/basic/build_defaults.xml
similarity index 100%
rename from externals/pio2/examples/basic/build_defaults.xml
rename to src/externals/pio2/examples/basic/build_defaults.xml
diff --git a/externals/pio2/examples/basic/check_mod.F90 b/src/externals/pio2/examples/basic/check_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/check_mod.F90
rename to src/externals/pio2/examples/basic/check_mod.F90
diff --git a/externals/pio2/examples/basic/config_bench.xml b/src/externals/pio2/examples/basic/config_bench.xml
similarity index 100%
rename from externals/pio2/examples/basic/config_bench.xml
rename to src/externals/pio2/examples/basic/config_bench.xml
diff --git a/externals/pio2/examples/basic/fdepends.awk b/src/externals/pio2/examples/basic/fdepends.awk
similarity index 100%
rename from externals/pio2/examples/basic/fdepends.awk
rename to src/externals/pio2/examples/basic/fdepends.awk
diff --git a/externals/pio2/examples/basic/gdecomp_mod.F90 b/src/externals/pio2/examples/basic/gdecomp_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/gdecomp_mod.F90
rename to src/externals/pio2/examples/basic/gdecomp_mod.F90
diff --git a/externals/pio2/examples/basic/kinds_mod.F90 b/src/externals/pio2/examples/basic/kinds_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/kinds_mod.F90
rename to src/externals/pio2/examples/basic/kinds_mod.F90
diff --git a/externals/pio2/examples/basic/kraken.128.csh b/src/externals/pio2/examples/basic/kraken.128.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.128.csh
rename to src/externals/pio2/examples/basic/kraken.128.csh
diff --git a/externals/pio2/examples/basic/kraken.1K.csh b/src/externals/pio2/examples/basic/kraken.1K.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.1K.csh
rename to src/externals/pio2/examples/basic/kraken.1K.csh
diff --git a/externals/pio2/examples/basic/kraken.256.csh b/src/externals/pio2/examples/basic/kraken.256.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.256.csh
rename to src/externals/pio2/examples/basic/kraken.256.csh
diff --git a/externals/pio2/examples/basic/kraken.512.csh b/src/externals/pio2/examples/basic/kraken.512.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.512.csh
rename to src/externals/pio2/examples/basic/kraken.512.csh
diff --git a/externals/pio2/examples/basic/kraken.64.csh b/src/externals/pio2/examples/basic/kraken.64.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.64.csh
rename to src/externals/pio2/examples/basic/kraken.64.csh
diff --git a/externals/pio2/examples/basic/namelist_mod.F90 b/src/externals/pio2/examples/basic/namelist_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/namelist_mod.F90
rename to src/externals/pio2/examples/basic/namelist_mod.F90
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.apb05 b/src/externals/pio2/examples/basic/namelists/testpio_in.apb05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.apb05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.apb05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.asb01 b/src/externals/pio2/examples/basic/namelists/testpio_in.asb01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.asb01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.asb01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.asb04 b/src/externals/pio2/examples/basic/namelists/testpio_in.asb04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.asb04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.asb04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b01 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b02 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b03 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b04 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b05 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b06 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b07 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b08 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4n01 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4n01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4n01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4n01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4n02 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4n02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4n02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4n02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4n03 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4n03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4n03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4n03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb01 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb02 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb03 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb04 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb05 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb06 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb07 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb08 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pn01 b/src/externals/pio2/examples/basic/namelists/testpio_in.pn01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pn01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pn01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pn02 b/src/externals/pio2/examples/basic/namelists/testpio_in.pn02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pn02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pn02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pn03 b/src/externals/pio2/examples/basic/namelists/testpio_in.pn03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pn03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pn03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps01 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps02 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps03 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps04 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps05 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps06 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps07 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps08 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb01 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb02 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb03 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb04 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb05 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb06 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb07 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb08 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sn01 b/src/externals/pio2/examples/basic/namelists/testpio_in.sn01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sn01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sn01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sn02 b/src/externals/pio2/examples/basic/namelists/testpio_in.sn02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sn02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sn02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sn03 b/src/externals/pio2/examples/basic/namelists/testpio_in.sn03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sn03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sn03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.wr01 b/src/externals/pio2/examples/basic/namelists/testpio_in.wr01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.wr01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.wr01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.wr02 b/src/externals/pio2/examples/basic/namelists/testpio_in.wr02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.wr02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.wr02
diff --git a/externals/pio2/examples/basic/perl5lib/ChangeLog b/src/externals/pio2/examples/basic/perl5lib/ChangeLog
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/ChangeLog
rename to src/externals/pio2/examples/basic/perl5lib/ChangeLog
diff --git a/externals/pio2/examples/basic/perl5lib/README b/src/externals/pio2/examples/basic/perl5lib/README
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/README
rename to src/externals/pio2/examples/basic/perl5lib/README
diff --git a/externals/pio2/examples/basic/perl5lib/XML/Changes b/src/externals/pio2/examples/basic/perl5lib/XML/Changes
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/Changes
rename to src/externals/pio2/examples/basic/perl5lib/XML/Changes
diff --git a/externals/pio2/examples/basic/perl5lib/XML/Lite.pm b/src/externals/pio2/examples/basic/perl5lib/XML/Lite.pm
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/Lite.pm
rename to src/externals/pio2/examples/basic/perl5lib/XML/Lite.pm
diff --git a/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm b/src/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm
rename to src/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm
diff --git a/externals/pio2/examples/basic/perl5lib/XML/README b/src/externals/pio2/examples/basic/perl5lib/XML/README
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/README
rename to src/externals/pio2/examples/basic/perl5lib/XML/README
diff --git a/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 b/src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3
rename to src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3
diff --git a/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 b/src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3
rename to src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3
diff --git a/externals/pio2/examples/basic/test.csh b/src/externals/pio2/examples/basic/test.csh
similarity index 100%
rename from externals/pio2/examples/basic/test.csh
rename to src/externals/pio2/examples/basic/test.csh
diff --git a/externals/pio2/examples/basic/testdecomp.F90 b/src/externals/pio2/examples/basic/testdecomp.F90
similarity index 100%
rename from externals/pio2/examples/basic/testdecomp.F90
rename to src/externals/pio2/examples/basic/testdecomp.F90
diff --git a/externals/pio2/examples/basic/testdecomp.bluefire.run b/src/externals/pio2/examples/basic/testdecomp.bluefire.run
similarity index 100%
rename from externals/pio2/examples/basic/testdecomp.bluefire.run
rename to src/externals/pio2/examples/basic/testdecomp.bluefire.run
diff --git a/externals/pio2/examples/basic/testdecomp_in b/src/externals/pio2/examples/basic/testdecomp_in
similarity index 100%
rename from externals/pio2/examples/basic/testdecomp_in
rename to src/externals/pio2/examples/basic/testdecomp_in
diff --git a/externals/pio2/examples/basic/testpio.F90 b/src/externals/pio2/examples/basic/testpio.F90
similarity index 100%
rename from externals/pio2/examples/basic/testpio.F90
rename to src/externals/pio2/examples/basic/testpio.F90
diff --git a/externals/pio2/examples/basic/testpio_bench.pl b/src/externals/pio2/examples/basic/testpio_bench.pl
similarity index 100%
rename from externals/pio2/examples/basic/testpio_bench.pl
rename to src/externals/pio2/examples/basic/testpio_bench.pl
diff --git a/externals/pio2/examples/basic/testpio_build.pl b/src/externals/pio2/examples/basic/testpio_build.pl
similarity index 100%
rename from externals/pio2/examples/basic/testpio_build.pl
rename to src/externals/pio2/examples/basic/testpio_build.pl
diff --git a/externals/pio2/examples/basic/testpio_run.pl b/src/externals/pio2/examples/basic/testpio_run.pl
similarity index 100%
rename from externals/pio2/examples/basic/testpio_run.pl
rename to src/externals/pio2/examples/basic/testpio_run.pl
diff --git a/externals/pio2/examples/basic/utils_mod.F90 b/src/externals/pio2/examples/basic/utils_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/utils_mod.F90
rename to src/externals/pio2/examples/basic/utils_mod.F90
diff --git a/externals/pio2/examples/basic/wstest.c b/src/externals/pio2/examples/basic/wstest.c
similarity index 100%
rename from externals/pio2/examples/basic/wstest.c
rename to src/externals/pio2/examples/basic/wstest.c
diff --git a/externals/pio2/examples/basic/ystest.sh b/src/externals/pio2/examples/basic/ystest.sh
similarity index 100%
rename from externals/pio2/examples/basic/ystest.sh
rename to src/externals/pio2/examples/basic/ystest.sh
diff --git a/src/externals/pio2/examples/c/CMakeLists.txt b/src/externals/pio2/examples/c/CMakeLists.txt
new file mode 100644
index 00000000000..74b4837c7f1
--- /dev/null
+++ b/src/externals/pio2/examples/c/CMakeLists.txt
@@ -0,0 +1,58 @@
+###-------------------------------------------------------------------------###
+### CMakeList.txt for examples using pio
+###-------------------------------------------------------------------------###
+
+# Include PIO include and lib directories
+INCLUDE_DIRECTORIES(${PIO_INCLUDE_DIRS})
+include_directories("${CMAKE_SOURCE_DIR}/examples/c")
+LINK_DIRECTORIES(${PIO_LIB_DIR})
+
+set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0")
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+endif()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
+
+ADD_EXECUTABLE(examplePio EXCLUDE_FROM_ALL examplePio.c)
+TARGET_LINK_LIBRARIES(examplePio pioc)
+add_dependencies(tests examplePio)
+
+ADD_EXECUTABLE(example1 example1.c)
+TARGET_LINK_LIBRARIES(example1 pioc)
+add_dependencies(tests example1)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME examplePio COMMAND examplePio)
+  add_test(NAME example1 COMMAND example1)
+else ()
+  add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60)
+  add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60)
+endif ()
+
+
+#===== MPE =====
+find_package (MPE "2.4.8" COMPONENTS C)
+if (PIO_ENABLE_FORTRAN)
+  if (MPE_C_FOUND)
+    SET(SRC example2.c)
+    ADD_EXECUTABLE(example2 ${SRC})
+    TARGET_LINK_LIBRARIES(example2 pioc)
+    target_include_directories (example2
+      PUBLIC ${MPE_C_INCLUDE_DIRS})
+    target_compile_definitions (example2
+      PUBLIC HAVE_MPE)
+    target_link_libraries (example2 pioc lmpe mpe pthread m)
+  endif ()
+endif ()
diff --git a/externals/pio2/examples/c/example1.c b/src/externals/pio2/examples/c/example1.c
similarity index 91%
rename from externals/pio2/examples/c/example1.c
rename to src/externals/pio2/examples/c/example1.c
index 8402c826125..1fcd8ab222a 100644
--- a/externals/pio2/examples/c/example1.c
+++ b/src/externals/pio2/examples/c/example1.c
@@ -195,21 +195,8 @@ int check_file(int ntasks, char *filename) {
 	 * classic format file (but with different libraries). The
 	 * last two produce netCDF4/HDF5 format files, written with
 	 * and without using netCDF-4 parallel I/O. */
-	int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, 
-					  PIO_IOTYPE_NETCDF,
-					  PIO_IOTYPE_NETCDF4C,
-					  PIO_IOTYPE_NETCDF4P};
-
-	/** Names for the output files. Two of them (pnetcdf and
-	 * classic) will be in classic netCDF format, the others
-	 * (serial4 and parallel4) will be in netCDF-4/HDF5
-	 * format. All four can be read by the netCDF library, and all
-	 * will contain the same contents. */
-	char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME] = {"example1_pnetcdf.nc",
-							  "example1_classic.nc",
-							  "example1_serial4.nc",
-							  "example1_parallel4.nc"};
-	
+	int format[NUM_NETCDF_FLAVORS];
+
 	/** Number of processors that will do IO. In this example we
 	 * will do IO from all processors. */
 	int niotasks;
@@ -218,15 +205,9 @@ int check_file(int ntasks, char *filename) {
 	 * example. */
 	int ioproc_stride = 1;
 
-	/** Number of the aggregator? Always 0 in this example. */
-	int numAggregator = 0;
-
 	/** Zero based rank of first processor to be used for I/O. */
 	int ioproc_start = 0;
 
-	/** Specifies the flavor of netCDF output format. */
-	int iotype;
-
 	/** The dimension ID. */
 	int dimid;
 
@@ -265,12 +246,6 @@ int check_file(int ntasks, char *filename) {
 	 * as elements_per_pe.*/
 	int *buffer;
 
-	/** A buffer for reading data back from the file. The size of
-	 * this array will vary depending on how many processors are
-	 * involved in the execution of the example code. It's length
-	 * will be the same as elements_per_pe.*/
-	int *read_buffer;
-
 	/** A 1-D array which holds the decomposition mapping for this
 	 * example. The size of this array will vary depending on how
 	 * many processors are involved in the execution of the
@@ -278,6 +253,12 @@ int check_file(int ntasks, char *filename) {
 	 * elements_per_pe. */
 	PIO_Offset *compdof;
 
+        /** Test filename. */
+        char filename[NC_MAX_NAME + 1];
+
+        /** The number of netCDF flavors available in this build. */
+        int num_flavors = 0;
+            
 	/** Used for command line processing. */
 	int c;
 
@@ -344,16 +325,29 @@ int check_file(int ntasks, char *filename) {
 				   compdof, &ioid, NULL, NULL, NULL)))
 	    ERR(ret);
 	free(compdof);
+
+        /* The number of favors may change with the build parameters. */
+#ifdef _PNETCDF
+        format[num_flavors++] = PIO_IOTYPE_PNETCDF;
+#endif
+        format[num_flavors++] = PIO_IOTYPE_NETCDF;
+#ifdef _NETCDF4
+        format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
+        format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
+#endif
 	
 	/* Use PIO to create the example file in each of the four
 	 * available ways. */
-	for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
+	for (int fmt = 0; fmt < num_flavors; fmt++) 
 	{
+            /* Create a filename. */
+            sprintf(filename, "example1_%d.nc", fmt);
+            
 	    /* Create the netCDF output file. */
 	    if (verbose)
 		printf("rank: %d Creating sample file %s with format %d...\n",
-		       my_rank, filename[fmt], format[fmt]);
-	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
+		       my_rank, filename, format[fmt]);
+	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename,
 				       PIO_CLOBBER)))
 		ERR(ret);
 	
@@ -406,9 +400,12 @@ int check_file(int ntasks, char *filename) {
 
 	/* Check the output file. */
 	if (!my_rank)
-	    for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
-		if ((ret = check_file(ntasks, filename[fmt])))
+	    for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                sprintf(filename, "example1_%d.nc", fmt);
+		if ((ret = check_file(ntasks, filename)))
 		    ERR(ret);
+            }
 
 	/* Finalize the MPI library. */
 	MPI_Finalize();
diff --git a/externals/pio2/examples/c/example2.c b/src/externals/pio2/examples/c/example2.c
similarity index 99%
rename from externals/pio2/examples/c/example2.c
rename to src/externals/pio2/examples/c/example2.c
index f4b75d075b7..becbcaccd29 100644
--- a/externals/pio2/examples/c/example2.c
+++ b/src/externals/pio2/examples/c/example2.c
@@ -48,8 +48,8 @@
  * responsibilty for writing and reading them will be spread between
  * all the processors used to run this example. */
 /**@{*/
-#define X_DIM_LEN 400
-#define Y_DIM_LEN 400
+#define X_DIM_LEN 20
+#define Y_DIM_LEN 30
 /**@}*/
 
 /** The number of timesteps of data to write. */
@@ -98,8 +98,7 @@ char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"};
 int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
 
 /** Length of chunksizes to use in netCDF-4 files. */
-size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
-
+PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
 
 /** Number of MPE events. The start and stop of each event will be
  * tracked, and graphed. This value is used outside of HAVE_MPE ifdefs.*/
diff --git a/externals/pio2/examples/c/examplePio.c b/src/externals/pio2/examples/c/examplePio.c
similarity index 97%
rename from externals/pio2/examples/c/examplePio.c
rename to src/externals/pio2/examples/c/examplePio.c
index d0820a7327d..297f3923ce5 100644
--- a/externals/pio2/examples/c/examplePio.c
+++ b/src/externals/pio2/examples/c/examplePio.c
@@ -163,18 +163,15 @@ typedef struct examplePioClass
  */
 struct examplePioClass* epc_init( struct examplePioClass* this )
 {
-    int ierr;
-    int argc;
-    char *argv;
     int i, localVal;
 
     /*
     ** initialize MPI
     */
     
-    ierr = MPI_Init(NULL, NULL);
-    ierr = MPI_Comm_rank(MPI_COMM_WORLD, &this->myRank);
-    ierr = MPI_Comm_size(MPI_COMM_WORLD, &this->ntasks);
+    MPI_Init(NULL, NULL);
+    MPI_Comm_rank(MPI_COMM_WORLD, &this->myRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &this->ntasks);
 
     /* Check that a valid number of processors was specified. */
     if (!(this->ntasks == 1 || this->ntasks == 2 || this->ntasks == 4 ||
@@ -235,10 +232,11 @@ struct examplePioClass* epc_init( struct examplePioClass* this )
         localVal++;
     }
 
-    if (this->verbose) {
-	printf("rank: %d length: %d [", this->myRank, this->arrIdxPerPe);
+    if (this->verbose)
+    {
+	printf("rank: %d length: %lld [", this->myRank, this->arrIdxPerPe);
 	for (i = 0; i < this->arrIdxPerPe; i++ ) {
-	    printf("%d", this->compdof[i]);
+	    printf("%lld", this->compdof[i]);
 	    if (i < this->arrIdxPerPe - 1)
 		printf(", ");
 	}
@@ -451,17 +449,15 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this )
  */
 struct examplePioClass* epc_cleanUp( struct examplePioClass* this )
 {
-    int ierr;
-    
     if (this->verbose)
 	printf("rank: %d Freeing local and library resources...\n", this->myRank);
     free(this->dataBuffer);
     free(this->readBuffer);
     free(this->compdof);
     
-    ierr = PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
-    ierr = PIOc_finalize(this->pioIoSystem);
-    ierr = MPI_Finalize();
+    PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
+    PIOc_finalize(this->pioIoSystem);
+    MPI_Finalize();
     
     return this;
 }
diff --git a/src/externals/pio2/examples/c/valsupp_example1.supp b/src/externals/pio2/examples/c/valsupp_example1.supp
new file mode 100644
index 00000000000..63f3e073836
--- /dev/null
+++ b/src/externals/pio2/examples/c/valsupp_example1.supp
@@ -0,0 +1,15 @@
+{
+   cond_jump_1
+   Memcheck:Cond
+   fun:MPIC_Waitall
+   fun:MPIR_Alltoallw_intra
+   fun:MPIR_Alltoallw
+   fun:MPIR_Alltoallw_impl
+   fun:PMPI_Alltoallw
+   fun:pio_swapm
+   fun:rearrange_comp2io
+   fun:PIOc_write_darray_multi
+   fun:flush_buffer
+   fun:PIOc_sync
+   fun:main
+}
\ No newline at end of file
diff --git a/externals/pio2/examples/cxx/CMakeLists.txt b/src/externals/pio2/examples/cxx/CMakeLists.txt
similarity index 100%
rename from externals/pio2/examples/cxx/CMakeLists.txt
rename to src/externals/pio2/examples/cxx/CMakeLists.txt
diff --git a/externals/pio2/examples/cxx/examplePio.cxx b/src/externals/pio2/examples/cxx/examplePio.cxx
similarity index 100%
rename from externals/pio2/examples/cxx/examplePio.cxx
rename to src/externals/pio2/examples/cxx/examplePio.cxx
diff --git a/externals/pio2/examples/f03/CMakeLists.txt b/src/externals/pio2/examples/f03/CMakeLists.txt
similarity index 96%
rename from externals/pio2/examples/f03/CMakeLists.txt
rename to src/externals/pio2/examples/f03/CMakeLists.txt
index e362ceacd13..a299c7cde8c 100644
--- a/externals/pio2/examples/f03/CMakeLists.txt
+++ b/src/externals/pio2/examples/f03/CMakeLists.txt
@@ -12,10 +12,9 @@ LINK_DIRECTORIES(${PIO_LIB_DIR})
 set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
-if(${PIO_BUILD_TIMING}) 
+if(${PIO_BUILD_TIMING})
   SET(TIMING_LINK_LIB timing)
 endif()
 SET(SRC examplePio.f90)
 ADD_EXECUTABLE(examplePio_f90 ${SRC})
 TARGET_LINK_LIBRARIES(examplePio_f90 piof pioc ${TIMING_LINK_LIB})
-
diff --git a/externals/pio2/examples/f03/examplePio.f90 b/src/externals/pio2/examples/f03/examplePio.f90
similarity index 99%
rename from externals/pio2/examples/f03/examplePio.f90
rename to src/externals/pio2/examples/f03/examplePio.f90
index f5eede9e26a..d2baddf2096 100644
--- a/externals/pio2/examples/f03/examplePio.f90
+++ b/src/externals/pio2/examples/f03/examplePio.f90
@@ -10,7 +10,7 @@ module pioExample
     use pio, only : PIO_nowrite, PIO_openfile
 
     implicit none
-    save
+
     private
 
     include 'mpif.h'
diff --git a/externals/pio2/scripts/prune_decomps.pl b/src/externals/pio2/scripts/prune_decomps.pl
similarity index 100%
rename from externals/pio2/scripts/prune_decomps.pl
rename to src/externals/pio2/scripts/prune_decomps.pl
diff --git a/externals/pio2/src/CMakeLists.txt b/src/externals/pio2/src/CMakeLists.txt
similarity index 72%
rename from externals/pio2/src/CMakeLists.txt
rename to src/externals/pio2/src/CMakeLists.txt
index ba6863d8509..5c5c079fa1e 100644
--- a/externals/pio2/src/CMakeLists.txt
+++ b/src/externals/pio2/src/CMakeLists.txt
@@ -2,11 +2,11 @@
 #  PRELIMINARIES
 #==============================================================================
 
-# Test for big-endian nature 
-if (PIO_TEST_BIG_ENDIAN) 
+# Test for big-endian nature
+if (PIO_TEST_BIG_ENDIAN)
   include (TestBigEndian)
   test_big_endian (PIO_BIG_ENDIAN_TEST_RESULT)
-  if (PIO_BIG_ENDIAN_TEST_RESULT) 
+  if (PIO_BIG_ENDIAN_TEST_RESULT)
     set (PIO_BIG_ENDIAN ON  CACHE BOOL "Whether machine is big endian")
   else ()
     set (PIO_BIG_ENDIAN OFF CACHE BOOL "Whether machine is big endian")
@@ -17,8 +17,8 @@ endif ()
 
 # Look for GPTL
 if (PIO_ENABLE_TIMING)
-    find_package (GPTL COMPONENTS C QUIET)
-    find_package (GPTL COMPONENTS Fortran_Perf QUIET)
+  find_package (GPTL COMPONENTS C QUIET)
+  find_package (GPTL COMPONENTS Fortran_Perf QUIET)
 endif ()
 
 #==============================================================================
@@ -27,13 +27,15 @@ endif ()
 
 # Build GPTL only if necessary
 if (PIO_ENABLE_TIMING)
-    if (NOT GPTL_C_FOUND OR NOT GPTL_Fortran_Perf_FOUND)
-        add_subdirectory (gptl)
-    endif ()
+  if (NOT GPTL_C_FOUND OR NOT GPTL_Fortran_Perf_FOUND)
+    add_subdirectory (gptl)
+  endif ()
 endif ()
 
 # Build the C library
 add_subdirectory (clib)
 
 # Build the Fortran library
-add_subdirectory (flib)
+if (PIO_ENABLE_FORTRAN)
+  add_subdirectory (flib)
+endif ()
diff --git a/src/externals/pio2/src/clib/CMakeLists.txt b/src/externals/pio2/src/clib/CMakeLists.txt
new file mode 100644
index 00000000000..d2c1c3bbeb6
--- /dev/null
+++ b/src/externals/pio2/src/clib/CMakeLists.txt
@@ -0,0 +1,167 @@
+cmake_minimum_required (VERSION 2.8.12)
+include (CheckFunctionExists)
+project (PIOC C)
+
+#==============================================================================
+#  DEFINE THE TARGET
+#==============================================================================
+
+add_library (pioc topology.c pio_file.c pioc_support.c pio_lists.c
+  pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c bget.c
+  pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c pio_varm.c
+  pio_darray.c pio_darray_int.c)
+
+# set up include-directories
+include_directories(
+  "${PROJECT_SOURCE_DIR}"   # to find foo/foo.h
+  "${PROJECT_BINARY_DIR}")  # to find foo/config.h
+
+# Include the clib source directory
+target_include_directories (pioc
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+# System and compiler CPP directives
+target_compile_definitions (pioc
+  PUBLIC ${CMAKE_SYSTEM_DIRECTIVE})
+target_compile_definitions (pioc
+  PUBLIC ${CMAKE_C_COMPILER_DIRECTIVE})
+
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (pioc
+    PRIVATE -std=c99)
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+  target_compile_options (pioc
+    PRIVATE -c99)
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+  target_compile_options (pioc
+    PRIVATE -std=c99)
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  target_compile_options (pioc
+    PRIVATE -std=c99)
+endif()
+
+#==============================================================================
+#  DEFINE THE INSTALL
+#==============================================================================
+
+# Library
+install (TARGETS pioc DESTINATION lib)
+
+# Include/Header File
+install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include)
+
+#==============================================================================
+#  DEFINE THE DEPENDENCIES
+#==============================================================================
+
+#===== MPI =====
+if (PIO_USE_MPISERIAL)
+  find_package (MPISERIAL COMPONENTS C REQUIRED)
+  if (MPISERIAL_C_FOUND)
+    target_compile_definitions (pioc
+      PRIVATE MPI_SERIAL)
+    target_include_directories (pioc
+      PUBLIC ${MPISERIAL_C_INCLUDE_DIRS})
+    target_link_libraries (pioc
+      PUBLIC ${MPISERIAL_C_LIBRARIES})
+
+    set (WITH_PNETCDF FALSE)
+  endif ()
+else ()
+  find_package (MPI REQUIRED)
+endif ()
+
+#===== GPTL =====
+if (PIO_ENABLE_TIMING)
+  find_package (GPTL COMPONENTS C QUIET)
+  if (GPTL_C_FOUND)
+    message (STATUS "Found GPTL C: ${GPTL_C_LIBRARIES}")
+    target_include_directories (pioc
+      PUBLIC ${GPTL_C_INCLUDE_DIRS})
+    target_link_libraries (pioc
+      PUBLIC ${GPTL_C_LIBRARIES})
+  else ()
+    message (STATUS "Using internal GPTL C library for timing")
+    target_link_libraries (pioc
+      PUBLIC gptl)
+  endif ()
+  target_compile_definitions (pioc
+    PUBLIC TIMING)
+endif ()
+
+#===== NetCDF-C =====
+find_package (NetCDF "4.3.3" COMPONENTS C)
+if (NetCDF_C_FOUND)
+  target_include_directories (pioc
+    PUBLIC ${NetCDF_C_INCLUDE_DIRS})
+  target_compile_definitions (pioc
+    PUBLIC _NETCDF)
+  target_link_libraries (pioc
+    PUBLIC ${NetCDF_C_LIBRARIES})
+  if (${NetCDF_C_HAS_PARALLEL})
+    target_compile_definitions (pioc
+      PUBLIC _NETCDF4)
+  endif ()
+  if (${NetCDF_C_LOGGING_ENABLED})
+    target_compile_definitions (pioc
+      PUBLIC NETCDF_C_LOGGING_ENABLED)
+    # netcdf.h needs this to be defined to use netCDF logging.
+    target_compile_definitions (pioc
+      PUBLIC LOGGING)
+  endif()
+else ()
+  target_compile_definitions (pioc
+    PUBLIC _NONETCDF)
+endif ()
+
+#===== PnetCDF-C =====
+if (WITH_PNETCDF)
+  find_package (PnetCDF "1.7.0" COMPONENTS C)
+endif ()
+if (PnetCDF_C_FOUND)
+  target_include_directories (pioc
+    PUBLIC ${PnetCDF_C_INCLUDE_DIRS})
+  target_compile_definitions (pioc
+    PUBLIC _PNETCDF)
+  target_link_libraries (pioc
+    PUBLIC ${PnetCDF_C_LIBRARIES})
+
+  # Check library for varn functions
+  set (CMAKE_REQUIRED_LIBRARIES ${PnetCDF_C_LIBRARY})
+  check_function_exists (ncmpi_get_varn PnetCDF_C_HAS_VARN)
+  if (PnetCDF_C_HAS_VARN)
+    target_compile_definitions(pioc
+      PUBLIC USE_PNETCDF_VARN
+      PUBLIC USE_PNETCDF_VARN_ON_READ)
+  endif()
+else ()
+  target_compile_definitions (pioc
+    PUBLIC _NOPNETCDF)
+endif ()
+
+#===== Add EXTRAs =====
+target_include_directories (pioc
+  PUBLIC ${PIO_C_EXTRA_INCLUDE_DIRS})
+target_link_libraries (pioc
+  PUBLIC ${PIO_C_EXTRA_LIBRARIES})
+target_compile_options (pioc
+  PRIVATE ${PIO_C_EXTRA_COMPILE_OPTIONS})
+target_compile_definitions (pioc
+  PUBLIC ${PIO_C_EXTRA_COMPILE_DEFINITIONS})
+if (PIO_C_EXTRA_LINK_FLAGS)
+  set_target_properties(pioc PROPERTIES
+    LINK_FLAGS ${PIO_C_EXTRA_LINK_FLAGS})
+endif ()
+
+#===== Check for necessities =====
+if (NOT PnetCDF_C_FOUND AND NOT NetCDF_C_FOUND)
+  message (FATAL_ERROR "Must have PnetCDF and/or NetCDF C libraries")
+endif ()
+
+include(CheckTypeSize)
+check_type_size("size_t" SIZEOF_SIZE_T)
+CHECK_TYPE_SIZE("long long" SIZEOF_LONG_LONG)
+if (NOT ${SIZEOF_SIZE_T} EQUAL ${SIZEOF_LONG_LONG})
+  message (FATAL_ERROR "size_t and long long must be the same size!")
+endif ()
diff --git a/src/externals/pio2/src/clib/bget.c b/src/externals/pio2/src/clib/bget.c
new file mode 100644
index 00000000000..726ce5580da
--- /dev/null
+++ b/src/externals/pio2/src/clib/bget.c
@@ -0,0 +1,1677 @@
+/*
+
+  B G E T
+
+  Buffer allocator
+
+  Designed and implemented in April of 1972 by John Walker, based on the
+  Case Algol OPRO$ algorithm implemented in 1966.
+
+  Reimplemented in 1975 by John Walker for the Interdata 70.
+  Reimplemented in 1977 by John Walker for the Marinchip 9900.
+  Reimplemented in 1982 by Duff Kurland for the Intel 8080.
+
+  Portable C version implemented in September of 1990 by an older, wiser
+  instance of the original implementor.
+
+  Souped up and/or weighed down  slightly  shortly  thereafter  by  Greg
+  Lutz.
+
+  AMIX  edition, including the new compaction call-back option, prepared
+  by John Walker in July of 1992.
+
+  Bug in built-in test program fixed, ANSI compiler warnings eradicated,
+  buffer pool validator  implemented,  and  guaranteed  repeatable  test
+  added by John Walker in October of 1995.
+
+  This program is in the public domain.
+
+  1. This is the book of the generations of Adam.   In the day that God
+  created man, in the likeness of God made he him;
+  2. Male and female created he them;  and  blessed     them,  and  called
+  their name Adam, in the day when they were created.
+  3. And  Adam  lived  an hundred and thirty years,     and begat a son in
+  his own likeness, and after his image; and called his name Seth:
+  4. And the days of  Adam  after  he  had  begotten  Seth  were  eight
+  hundred years: and he begat sons and daughters:
+  5. And  all  the  days  that Adam lived were nine     hundred and thirty
+  years: and he died.
+  6. And Seth lived an hundred and five years, and begat Enos:
+  7. And Seth lived after he begat Enos eight hundred and seven  years,
+  and begat sons and daughters:
+  8.  And  all the days of Seth were nine hundred and twelve years: and
+  he died.
+  9. And Enos lived ninety years, and begat Cainan:
+  10. And Enos lived after he begat  Cainan eight  hundred  and  fifteen
+  years, and begat sons and daughters:
+  11. And  all  the days of Enos were nine hundred  and five years:  and
+  he died.
+  12. And Cainan lived seventy years and begat Mahalaleel:
+  13. And Cainan lived  after he  begat  Mahalaleel  eight  hundred  and
+  forty years, and begat sons and daughters:
+  14. And  all the days of Cainan were nine  hundred and ten years:  and
+  he died.
+  15. And Mahalaleel lived sixty and five years, and begat Jared:
+  16. And Mahalaleel lived  after  he  begat    Jared  eight  hundred  and
+  thirty years, and begat sons and daughters:
+  17. And  all  the  days  of Mahalaleel  were eight hundred    ninety and
+  five years: and he died.
+  18. And Jared lived an hundred sixty and  two  years,   and  he  begat
+  Enoch:
+  19. And  Jared  lived  after he begat Enoch  eight hundred years,  and
+  begat sons and daughters:
+  20. And all the days of Jared  were nine hundred sixty and two  years:
+  and he died.
+  21. And Enoch lived sixty and five years, and begat Methuselah:
+  22. And  Enoch  walked   with  God    after  he  begat Methuselah  three
+  hundred years, and begat sons and daughters:
+  23. And all the days of  Enoch  were  three  hundred  sixty  and  five
+  years:
+  24. And Enoch walked with God: and he was not; for God took him.
+  25. And  Methuselah  lived    an  hundred  eighty and  seven years,  and
+  begat Lamech.
+  26. And Methuselah lived after he  begat Lamech seven  hundred  eighty
+  and two years, and begat sons and daughters:
+  27. And  all the days of Methuselah  were nine hundred  sixty and nine
+  years: and he died.
+  28. And Lamech lived an hundred eighty  and two  years,  and  begat  a
+  son:
+  29. And  he called his name Noah, saying,  This same shall    comfort us
+  concerning  our  work and toil of our hands, because of the ground
+  which the LORD hath cursed.
+  30. And  Lamech  lived  after  he begat Noah  five hundred    ninety and
+  five years, and begat sons and daughters:
+  31. And all the days of Lamech were  seven hundred seventy    and  seven
+  years: and he died.
+  32. And  Noah  was five hundred years old:    and Noah begat Shem,  Ham,
+  and Japheth.
+
+  And buffers begat buffers, and links begat    links,  and  buffer  pools
+  begat  links  to chains of buffer pools containing buffers, and lo the
+  buffers and links and pools of buffers and pools of links to chains of
+  pools  of  buffers were fruitful and they multiplied and the Operating
+  System looked down upon them and said that it was Good.
+
+
+  INTRODUCTION
+  ============
+
+  BGET  is a comprehensive memory allocation package which is easily
+  configured to the needs of an application.    BGET is  efficient  in
+  both  the  time  needed to allocate and release buffers and in the
+  memory  overhead  required    for  buffer   pool   management.    It
+  automatically    consolidates   contiguous     space   to   minimise
+  fragmentation.  BGET is configured    by  compile-time  definitions,
+  Major options include:
+
+  *   A  built-in  test  program        to  exercise  BGET   and
+  demonstrate how the various functions are used.
+
+  *   Allocation  by  either the "first fit" or "best fit"
+  method.
+
+  *   Wiping buffers at release time to catch  code  which
+  references previously released storage.
+
+  *   Built-in  routines to dump individual buffers or the
+  entire buffer pool.
+
+  *   Retrieval of allocation and pool size statistics.
+
+  *   Quantisation of buffer sizes to a power  of  two  to
+  satisfy hardware alignment constraints.
+
+  *   Automatic  pool compaction, growth, and shrinkage by
+  means of call-backs to user defined functions.
+
+  Applications  of  BGET  can  range    from  storage  management   in
+  ROM-based  embedded programs to providing the framework upon which
+  a  multitasking  system  incorporating   garbage   collection   is
+  constructed.   BGET  incorporates  extensive  internal consistency
+  checking using the  mechanism; all these checks  can  be
+  turned off by compiling with NDEBUG defined, yielding a version of
+  BGET with minimal size and maximum speed.
+
+  The  basic    algorithm  underlying  BGET  has withstood the test of
+  time;  more  than  25  years   have   passed   since   the     first
+  implementation  of    this  code.  And yet, it is substantially more
+  efficient than the native allocation  schemes  of  many  operating
+  systems: the Macintosh and Microsoft Windows to name two, on which
+  programs have obtained substantial speed-ups by layering  BGET  as
+  an application level memory manager atop the underlying system's.
+
+  BGET has been implemented on the largest mainframes and the lowest
+  of    microprocessors.   It  has served as the core for multitasking
+  operating systems, multi-thread applications, embedded software in
+  data  network switching processors, and a host of C programs.  And
+  while it has accreted flexibility and additional options over  the
+  years,  it    remains  fast, memory efficient, portable, and easy to
+  integrate into your program.
+
+
+  BGET IMPLEMENTATION ASSUMPTIONS
+  ===============================
+
+  BGET is written in as portable a dialect of C  as  possible.   The
+  only   fundamental     assumption   about  the  underlying  hardware
+  architecture is that memory is allocated is a linear  array  which
+  can  be  addressed  as a vector of C "char" objects.  On segmented
+  address space architectures, this generally means that BGET should
+  be used to allocate storage within a single segment (although some
+  compilers     simulate   linear   address   spaces   on    segmented
+  architectures).   On  segmented  architectures,  then, BGET buffer
+  pools  may not be larger than a segment, but since BGET allows any
+  number of separate buffer pools, there is no limit    on  the  total
+  storage  which  can  be  managed,  only  on the largest individual
+  object which can be allocated.  Machines  with  a  linear  address
+  architecture,  such  as  the VAX, 680x0, Sparc, MIPS, or the Intel
+  80386 and above in native mode, may use BGET without restriction.
+
+
+  GETTING STARTED WITH BGET
+  =========================
+
+  Although BGET can be configured in a multitude of fashions,  there
+  are  three    basic  ways  of  working  with  BGET.   The  functions
+  mentioned below are documented in the following  section.     Please
+  excuse  the  forward  references which are made in the interest of
+  providing a roadmap to guide you  to  the  BGET  functions  you're
+  likely to need.
+
+  Embedded Applications
+  ---------------------
+
+  Embedded applications  typically  have  a  fixed  area  of    memory
+  dedicated  to  buffer  allocation (often in a separate RAM address
+  space distinct from the ROM that contains  the  executable    code).
+  To    use  BGET in such an environment, simply call bpool() with the
+  start address and length of the buffer  pool  area    in  RAM,  then
+  allocate  buffers  with  bget()  and  release  them  with  brel().
+  Embedded applications with very limited RAM but abundant CPU speed
+  may  benefit  by configuring BGET for BestFit allocation (which is
+  usually not worth it in other environments).
+
+  Malloc() Emulation
+  ------------------
+
+  If the C library malloc() function is too  slow,  not  present  in
+  your  development environment (for example, an a native Windows or
+  Macintosh program), or otherwise unsuitable, you  can  replace  it
+  with  BGET.  Initially define a buffer pool of an appropriate size
+  with bpool()--usually obtained by making a call to    the  operating
+  system's  low-level  memory allocator.  Then allocate buffers with
+  bget(), bgetz(), and bgetr() (the last two permit  the  allocation
+  of    buffers initialised to zero and [inefficient] re-allocation of
+  existing buffers for  compatibility  with  C  library  functions).
+  Release buffers by calling brel().    If a buffer allocation request
+  fails, obtain more storage from the underlying  operating  system,
+  add it to the buffer pool by another call to bpool(), and continue
+  execution.
+
+  Automatic Storage Management
+  ----------------------------
+
+  You can use BGET as your application's native memory  manager  and
+  implement  automatic  storage  pool  expansion,  contraction,  and
+  optionally application-specific  memory  compaction  by  compiling
+  BGET  with    the  BECtl  variable defined, then calling bectl() and
+  supplying  functions  for  storage    compaction,  acquisition,  and
+  release,  as  well as a standard pool expansion increment.    All of
+  these functions are optional (although it doesn't make much  sense
+  to    provide  a  release  function without an acquisition function,
+  does it?).    Once the call-back functions have  been  defined  with
+  bectl(),  you simply use bget() and brel() to allocate and release
+  storage as before.    You can supply an  initial  buffer  pool  with
+  bpool()  or  rely  on  automatic  allocation to acquire the entire
+  pool.  When a call on  bget()  cannot  be  satisfied,  BGET  first
+  checks  if    a compaction function has been supplied.  If so, it is
+  called (with the space required to satisfy the allocation  request
+  and a sequence number to allow the compaction routine to be called
+  successively without looping).  If the compaction function is able
+  to  free any storage (it needn't know whether the storage it freed
+  was adequate) it should return a  nonzero  value,  whereupon  BGET
+  will retry the allocation request and, if it fails again, call the
+  compaction function again with the next-higher sequence number.
+
+  If    the  compaction  function  returns zero, indicating failure to
+  free space, or no compaction function is defined, BGET next  tests
+  whether  a    non-NULL  allocation function was supplied to bectl().
+  If so, that function is called with  an  argument  indicating  how
+  many  bytes  of  additional  space are required.  This will be the
+  standard pool expansion increment supplied in the call to  bectl()
+  unless  the  original  bget()  call requested a buffer larger than
+  this; buffers larger than the standard pool block can  be  managed
+  "off  the books" by BGET in this mode.  If the allocation function
+  succeeds in obtaining the storage, it returns a pointer to the new
+  block  and    BGET  expands  the  buffer  pool;  if  it  fails,  the
+  allocation request fails and returns NULL to  the  caller.     If  a
+  non-NULL  release  function  is  supplied,    expansion blocks which
+  become totally empty are released  to  the    global  free  pool  by
+  passing their addresses to the release function.
+
+  Equipped  with  appropriate  allocation,  release,    and compaction
+  functions, BGET can be used as part of very  sophisticated    memory
+  management     strategies,  including  garbage  collection.   (Note,
+  however, that BGET is *not* a garbage  collector  by  itself,  and
+  that  developing  such a system requires much additional logic and
+  careful design of the application's memory allocation strategy.)
+
+
+  BGET FUNCTION DESCRIPTIONS
+  ==========================
+
+  Functions implemented in this file (some are enabled by certain of
+  the optional settings below):
+
+  void bpool(void *buffer, bufsize len);
+
+  Create a buffer pool of  bytes, using the storage starting at
+  .     You  can  call  bpool()  subsequently  to   contribute
+  additional storage to the overall buffer pool.
+
+  void *bget(bufsize size);
+
+  Allocate  a  buffer of  bytes.  The address of the buffer is
+  returned, or NULL if insufficient memory was available to allocate
+  the buffer.
+
+  void *bgetz(bufsize size);
+
+  Allocate a buffer of  bytes and clear it to all zeroes.  The
+  address of the buffer is returned, or NULL if insufficient    memory
+  was available to allocate the buffer.
+
+  void *bgetr(void *buffer, bufsize newsize);
+
+  Reallocate a buffer previously allocated by bget(),  changing  its
+  size  to    and  preserving  all  existing data.  NULL is
+  returned if insufficient memory is    available  to  reallocate  the
+  buffer, in which case the original buffer remains intact.
+
+  void brel(void *buf);
+
+  Return  the  buffer  , previously allocated by bget(), to the
+  free space pool.
+
+  void bectl(int (*compact)(bufsize sizereq, int sequence),
+  void *(*acquire)(bufsize size),
+  void (*release)(void *buf),
+  bufsize pool_incr);
+
+  Expansion control: specify functions through which the package may
+  compact  storage  (or  take  other    appropriate  action)  when  an
+  allocation    request  fails,  and  optionally automatically acquire
+  storage for expansion blocks  when    necessary,  and  release  such
+  blocks when they become empty.  If  is non-NULL, whenever
+  a buffer allocation request fails, the  function will  be
+  called with arguments specifying the number of bytes (total buffer
+  size,  including  header  overhead)  required   to     satisfy   the
+  allocation request, and a sequence number indicating the number of
+  consecutive  calls    on    attempting  to   satisfy   this
+  allocation    request.   The sequence number is 1 for the first call
+  on  for a given allocation  request,  and    increments  on
+  subsequent    calls,  permitting  the    function  to  take
+  increasingly dire measures in an attempt to free up  storage.   If
+  the    function  returns  a nonzero value, the allocation
+  attempt is re-tried.  If  returns 0 (as  it  must    if  it
+  isn't  able  to  release  any  space  or add storage to the buffer
+  pool), the allocation request fails, which can  trigger  automatic
+  pool expansion if the  argument is non-NULL.  At the time
+  the    function  is  called,  the    state  of  the  buffer
+  allocator  is  identical  to  that    at  the  moment the allocation
+  request was made; consequently, the   function  may  call
+  brel(), bpool(), bstats(), and/or directly manipulate  the    buffer
+  pool  in  any  manner which would be valid were the application in
+  control.  This does not, however, relieve the    function
+  of the need to ensure that whatever actions it takes do not change
+  things   underneath  the  application  that  made  the  allocation
+  request.  For example, a  function that released a buffer
+  in    the  process  of  being reallocated with bgetr() would lead to
+  disaster.  Implementing a safe and effective    mechanism
+  requires  careful  design of an application's memory architecture,
+  and cannot generally be easily retrofitted into existing code.
+
+  If  is non-NULL, that function will be called whenever an
+  allocation    request  fails.  If the  function succeeds in
+  allocating the requested space and returns a pointer  to  the  new
+  area,  allocation will proceed using the expanded buffer pool.  If
+   cannot obtain the requested space, it should return NULL
+  and   the     entire  allocation  process  will  fail.   
+  specifies the normal expansion block size.    Providing an 
+  function will cause subsequent bget()  requests  for  buffers  too
+  large  to  be  managed in the linked-block scheme (in other words,
+  larger than  minus the buffer overhead) to be satisfied
+  directly by calls to the  function.  Automatic release of
+  empty pool blocks will occur only if all pool blocks in the system
+  are the size given by .
+
+  void bstats(bufsize *curalloc, bufsize *totfree,
+  bufsize *maxfree, long *nget, long *nrel);
+
+  The amount    of  space  currently  allocated  is  stored  into  the
+  variable  pointed  to by .  The total free space (sum of
+  all free blocks in the pool) is stored into the  variable  pointed
+  to    by  , and the size of the largest single block in the
+  free space    pool  is  stored  into  the  variable  pointed  to  by
+  .     The  variables  pointed  to  by  and  are
+  filled, respectively, with    the  number  of  successful  (non-NULL
+  return) bget() calls and the number of brel() calls.
+
+  void bstatse(bufsize *pool_incr, long *npool,
+  long *npget, long *nprel,
+  long *ndget, long *ndrel);
+
+  Extended  statistics: The expansion block size will be stored into
+  the variable pointed to by , or the negative thereof if
+  automatic  expansion  block  releases are disabled.  The number of
+  currently active pool blocks will  be  stored  into  the  variable
+  pointed  to  by  .  The variables pointed to by  and
+   will be filled with, respectively, the number of expansion
+  block   acquisitions   and    releases  which  have  occurred.   The
+  variables pointed to by  and  will  be  filled  with
+  the  number  of  bget()  and  brel()  calls, respectively, managed
+  through blocks directly allocated by the acquisition  and  release
+  functions.
+
+  void bufdump(void *buf);
+
+  The buffer pointed to by  is dumped on standard output.
+
+  void bpoold(void *pool, int dumpalloc, int dumpfree);
+
+  All buffers in the buffer pool , previously initialised by a
+  call on bpool(), are listed in ascending memory address order.  If
+   is nonzero, the  contents  of  allocated  buffers  are
+  dumped;  if  is nonzero, the contents of free blocks are
+  dumped.
+
+  int bpoolv(void *pool);
+
+  The  named    buffer  pool,  previously  initialised  by  a  call on
+  bpool(), is validated for bad pointers, overwritten data, etc.  If
+  compiled with NDEBUG not defined, any error generates an assertion
+  failure.  Otherwise 1 is returned if the pool is valid,  0    if  an
+  error is found.
+
+
+  BGET CONFIGURATION
+  ==================
+*/
+#include 
+#include 
+#include 
+#ifdef PIO_USE_MALLOC
+#include 
+#endif
+
+#define TestProg    20000             /* Generate built-in test program
+                                         if defined.  The value specifies
+                                         how many buffer allocation attempts
+                                         the test program should make. */
+#undef TestProg
+
+#define SizeQuant   4                 /* Buffer allocation size quantum:
+                                         all buffers allocated are a
+                                         multiple of this size.  This
+                                         MUST be a power of two. */
+
+#define BufDump     1                 /* Define this symbol to enable the
+                                         bpoold() function which dumps the
+                                         buffers in a buffer pool. */
+
+#define BufValid    1                 /* Define this symbol to enable the
+                                         bpoolv() function for validating
+                                         a buffer pool. */
+
+#define DumpData    1                 /* Define this symbol to enable the
+                                         bufdump() function which allows
+                                         dumping the contents of an allocated
+                                         or free buffer. */
+
+#define BufStats    1                 /* Define this symbol to enable the
+                                         bstats() function which calculates
+                                         the total free space in the buffer
+                                         pool, the largest available
+                                         buffer, and the total space
+                                         currently allocated. */
+
+#define FreeWipe    1                 /* Wipe free buffers to a guaranteed
+                                         pattern of garbage to trip up
+                                         miscreants who attempt to use
+                                         pointers into released buffers. */
+
+//#define BestFit     1
+#undef BestFit
+/* Use a best fit algorithm when
+   searching for space for an
+   allocation request.  This uses
+   memory more efficiently, but
+   allocation will be much slower. */
+
+#define BECtl       1                 /* Define this symbol to enable the
+                                         bectl() function for automatic
+                                         pool space control.  */
+
+#include 
+
+#ifdef lint
+#define NDEBUG                        /* Exits in asserts confuse lint */
+/* LINTLIBRARY */                     /* Don't complain about def, no ref */
+extern char *sprintf();               /* Sun includes don't define sprintf */
+#endif
+
+#include 
+#include 
+
+#ifdef BufDump                        /* BufDump implies DumpData */
+#ifndef DumpData
+#define DumpData    1
+#endif
+#endif
+
+#ifdef DumpData
+#include 
+#endif
+
+/*  Declare the interface, including the requested buffer size type,
+    bufsize.  */
+
+#include "bget.h"
+
+#define MemSize     size_t            /* Type for size arguments to memxxx()
+                                         functions such as memcmp(). */
+
+/* Queue links */
+
+struct qlinks {
+    struct bfhead *flink;             /* Forward link */
+    struct bfhead *blink;             /* Backward link */
+};
+
+/* Header in allocated and free buffers */
+
+struct bhead {
+    bufsize prevfree;                 /* Relative link back to previous
+                                         free buffer in memory or 0 if
+                                         previous buffer is allocated.  */
+    bufsize bsize;                    /* Buffer size: positive if free,
+                                         negative if allocated. */
+};
+#define BH(p)   ((struct bhead *) (p))
+
+/*  Header in directly allocated buffers (by acqfcn) */
+
+struct bdhead {
+    bufsize tsize;                    /* Total size, including overhead */
+    struct bhead bh;                  /* Common header */
+};
+#define BDH(p)  ((struct bdhead *) (p))
+
+/* Header in free buffers */
+
+struct bfhead {
+    struct bhead bh;                  /* Common allocated/free header */
+    struct qlinks ql;                 /* Links on free list */
+};
+#define BFH(p)  ((struct bfhead *) (p))
+
+static struct bfhead freelist = {     /* List of free buffers */
+    {0, 0},
+    {&freelist, &freelist}
+};
+
+
+#ifdef BufStats
+static bufsize totalloc = 0;          /* Total space currently allocated */
+static long numget = 0, numrel = 0;   /* Number of bget() and brel() calls */
+#ifdef BECtl
+static long numpblk = 0;              /* Number of pool blocks */
+static long numpget = 0, numprel = 0; /* Number of block gets and rels */
+static long numdget = 0, numdrel = 0; /* Number of direct gets and rels */
+#endif /* BECtl */
+#endif /* BufStats */
+
+#ifdef BECtl
+
+/* Automatic expansion block management functions */
+
+static int (*compfcn) _((bufsize sizereq, int sequence)) = NULL;
+static void *(*acqfcn) _((bufsize size)) = NULL;
+static void (*relfcn) _((void *buf)) = NULL;
+
+static bufsize exp_incr = 0;          /* Expansion block size */
+static bufsize pool_len = 0;          /* 0: no bpool calls have been made
+                                         -1: not all pool blocks are
+                                         the same size
+                                         >0: (common) block size for all
+                                         bpool calls made so far
+                                      */
+#endif
+
+/*  Minimum allocation quantum: */
+
+#define QLSize  (sizeof(struct qlinks))
+#define SizeQ   ((SizeQuant > QLSize) ? SizeQuant : QLSize)
+
+#define V   (void)                    /* To denote unwanted returned values */
+
+/* End sentinel: value placed in bsize field of dummy block delimiting
+   end of pool block.  The most negative number which will  fit  in  a
+   bufsize, defined in a way that the compiler will accept. */
+
+#define ESent   ((bufsize) (-(((1L << (sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2))
+
+/* added for PIO so that a bpool can be freed and another allocated */
+void bpoolrelease()
+{
+    LOG((2, "bpoolrelease"));
+    freelist.bh.prevfree=0;
+    freelist.bh.bsize=0;
+    freelist.ql.flink=&freelist;
+    freelist.ql.blink=&freelist;
+    LOG((2, "bpoolrelease"));
+
+#ifdef BufStats
+    totalloc = 0;             /* Total space currently allocated */
+    numget = 0;
+    numrel = 0;   /* Number of bget() and brel() calls */
+#ifdef BECtl
+    numpblk = 0;              /* Number of pool blocks */
+    numpget = 0;
+    numprel = 0; /* Number of block gets and rels */
+    numdget = 0;
+    numdrel = 0; /* Number of direct gets and rels */
+#endif /* BECtl */
+#endif /* BufStats */
+    LOG((2, "bpoolrelease"));
+
+#ifdef BECtl
+/* Automatic expansion block management functions */
+    compfcn = NULL;
+    acqfcn = NULL;
+    relfcn = NULL;
+    exp_incr = 0;
+    pool_len = 0;
+#endif
+    LOG((2, "bpoolrelease"));
+
+}
+
+
+
+/*  BGET  --  Allocate a buffer.  */
+
+void *bget(requested_size)
+bufsize requested_size;
+{
+    bufsize size = requested_size;
+    struct bfhead *b;
+#ifdef BestFit
+    struct bfhead *best;
+#endif
+    void *buf;
+#ifdef BECtl
+    int compactseq = 0;
+#endif
+
+#ifdef PIO_USE_MALLOC
+    //    if(requested_size>maxsize){
+    //   maxsize=requested_size;
+    //   printf("%s %d %d\n",__FILE__,__LINE__,maxsize);
+    // }
+    buf = malloc(requested_size);
+    //    printf("bget allocate %ld %x\n",requested_size,buf);
+    return(buf);
+#endif
+
+
+    if(size<=0)
+        print_trace(NULL);
+
+    assert(size > 0);
+
+    if (size < SizeQ) {               /* Need at least room for the */
+        size = SizeQ;                 /*    queue links.  */
+    }
+#ifdef SizeQuant
+#if SizeQuant > 1
+    size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
+#endif
+#endif
+
+    size += sizeof(struct bhead);     /* Add overhead in allocated buffer
+                                         to size required. */
+
+#ifdef BECtl
+    /* If a compact function was provided in the call to bectl(), wrap
+       a loop around the allocation process  to  allow  compaction  to
+       intervene in case we don't find a suitable buffer in the chain. */
+
+    while (1) {
+#endif
+        b = freelist.ql.flink;
+#ifdef BestFit
+        best = &freelist;
+#endif
+
+
+        /* Scan the free list searching for the first buffer big enough
+           to hold the requested size buffer. */
+
+#ifdef BestFit
+        while (b != &freelist) {
+            //    printf("%s %d %X %X %X %ld \n",__FILE__,__LINE__,b,&freelist,best,b->bh.bsize);
+            if (b->bh.bsize >= size) {
+                if ((best == &freelist) || (b->bh.bsize < best->bh.bsize)) {
+                    best = b;
+                }
+            }
+            b = b->ql.flink;              /* Link to next buffer */
+        }
+        b = best;
+#endif /* BestFit */
+
+        while (b != &freelist) {
+            if ((bufsize) b->bh.bsize >= size) {
+
+                /* Buffer  is big enough to satisfy  the request.  Allocate it
+                   to the caller.  We must decide whether the buffer is  large
+                   enough  to  split  into  the part given to the caller and a
+                   free buffer that remains on the free list, or  whether  the
+                   entire  buffer  should  be  removed  from the free list and
+                   given to the caller in its entirety.   We  only  split  the
+                   buffer if enough room remains for a header plus the minimum
+                   quantum of allocation. */
+
+                if ((b->bh.bsize - size) > (SizeQ + (sizeof(struct bhead)))) {
+                    struct bhead *ba, *bn;
+
+                    ba = BH(((char *) b) + (b->bh.bsize - size));
+                    bn = BH(((char *) ba) + size);
+                    assert(bn->prevfree == b->bh.bsize);
+                    /* Subtract size from length of free block. */
+                    b->bh.bsize -= size;
+                    /* Link allocated buffer to the previous free buffer. */
+                    ba->prevfree = b->bh.bsize;
+                    /* Plug negative size into user buffer. */
+                    ba->bsize = -(bufsize) size;
+                    /* Mark buffer after this one not preceded by free block. */
+                    bn->prevfree = 0;
+
+#ifdef BufStats
+                    totalloc += size;
+                    numget++;             /* Increment number of bget() calls */
+#endif
+                    buf = (void *) ((((char *) ba) + sizeof(struct bhead)));
+                    return buf;
+                } else {
+                    struct bhead *ba;
+
+                    ba = BH(((char *) b) + b->bh.bsize);
+                    assert(ba->prevfree == b->bh.bsize);
+
+                    /* The buffer isn't big enough to split.  Give  the  whole
+                       shebang to the caller and remove it from the free list. */
+
+                    assert(b->ql.blink->ql.flink == b);
+                    assert(b->ql.flink->ql.blink == b);
+                    b->ql.blink->ql.flink = b->ql.flink;
+                    b->ql.flink->ql.blink = b->ql.blink;
+
+#ifdef BufStats
+                    totalloc += b->bh.bsize;
+                    numget++;             /* Increment number of bget() calls */
+#endif
+                    /* Negate size to mark buffer allocated. */
+                    b->bh.bsize = -(b->bh.bsize);
+
+                    /* Zero the back pointer in the next buffer in memory
+                       to indicate that this buffer is allocated. */
+                    ba->prevfree = 0;
+
+                    /* Give user buffer starting at queue links. */
+                    buf =  (void *) &(b->ql);
+                    return buf;
+                }
+            }
+            b = b->ql.flink;              /* Link to next buffer */
+        }
+#ifdef BECtl
+
+        /* We failed to find a buffer.  If there's a compact  function
+           defined,  notify  it  of the size requested.  If it returns
+           TRUE, try the allocation again. */
+
+        if ((compfcn == NULL) || (!(*compfcn)(size, ++compactseq))) {
+            break;
+        }
+    }
+
+    /* No buffer available with requested size free. */
+
+    /* Don't give up yet -- look in the reserve supply. */
+
+    if (acqfcn != NULL) {
+        if (size > exp_incr - sizeof(struct bhead)) {
+
+            /* Request  is  too  large  to  fit in a single expansion
+               block.  Try to satisy it by a direct buffer acquisition. */
+
+            struct bdhead *bdh;
+
+            size += sizeof(struct bdhead) - sizeof(struct bhead);
+            if ((bdh = BDH((*acqfcn)((bufsize) size))) != NULL) {
+
+                /*  Mark the buffer special by setting the size field
+                    of its header to zero.  */
+                bdh->bh.bsize = 0;
+                bdh->bh.prevfree = 0;
+                bdh->tsize = size;
+#ifdef BufStats
+                totalloc += size;
+                numget++;             /* Increment number of bget() calls */
+                numdget++;            /* Direct bget() call count */
+#endif
+                buf =  (void *) (bdh + 1);
+
+                /*only let this happen once */
+                printf("%s %d memory request exceeds block size %ld %ld\n",__FILE__,__LINE__,
+                       size,exp_incr);
+                exp_incr = size+sizeof(struct bhead);
+
+                return buf;
+            }
+
+        } else {
+
+            /*  Try to obtain a new expansion block */
+
+            void *newpool;
+
+            if ((newpool = (*acqfcn)((bufsize) exp_incr)) != NULL) {
+                bpool(newpool, exp_incr);
+                buf =  bget(requested_size);  /* This can't, I say, can't
+                                                 get into a loop. */
+                //              printf("%s %d new memory block of size %d\n",__FILE__,__LINE__,exp_incr);
+                return buf;
+            }
+        }
+    }
+
+    /*  Still no buffer available */
+
+#endif /* BECtl */
+
+    return NULL;
+}
+
+/*  BGETZ  --  Allocate a buffer and clear its contents to zero.  We clear
+    the  entire  contents  of  the buffer to zero, not just the
+    region requested by the caller. */
+
+void *bgetz(size)
+bufsize size;
+{
+    char *buf = (char *) bget(size);
+
+    if (buf != NULL) {
+        struct bhead *b;
+        bufsize rsize;
+
+        b = BH(buf - sizeof(struct bhead));
+        rsize = -(b->bsize);
+        if (rsize == 0) {
+            struct bdhead *bd;
+
+            bd = BDH(buf - sizeof(struct bdhead));
+            rsize = bd->tsize - sizeof(struct bdhead);
+        } else {
+            rsize -= sizeof(struct bhead);
+        }
+        assert(rsize >= size);
+        V memset(buf, 0, (MemSize) rsize);
+    }
+    return ((void *) buf);
+}
+
+/*  BGETR  --  Reallocate a buffer.  This is a minimal implementation,
+    simply in terms of brel()  and  bget().      It  could  be
+    enhanced to allow the buffer to grow into adjacent free
+    blocks and to avoid moving data unnecessarily.  */
+
+void *bgetr(buf, size)
+void *buf;
+bufsize size;
+{
+    void *nbuf;
+    bufsize osize;                    /* Old size of buffer */
+    struct bhead *b;
+
+#ifdef PIO_USE_MALLOC
+    return(realloc(buf, size));
+#endif
+    if ((nbuf = bget(size)) == NULL) { /* Acquire new buffer */
+        return NULL;
+    }
+    if (buf == NULL) {
+        return nbuf;
+    }
+    b = BH(((char *) buf) - sizeof(struct bhead));
+    osize = -b->bsize;
+#ifdef BECtl
+    if (osize == 0) {
+        /*  Buffer acquired directly through acqfcn. */
+        struct bdhead *bd;
+
+        bd = BDH(((char *) buf) - sizeof(struct bdhead));
+        osize = bd->tsize - sizeof(struct bdhead);
+    } else
+#endif
+        osize -= sizeof(struct bhead);
+    assert(osize > 0);
+    V memcpy((char *) nbuf, (char *) buf, /* Copy the data */
+             (MemSize) ((size < osize) ? size : osize));
+    brel(buf);
+    return nbuf;
+}
+
+/*  BREL  --  Release a buffer.  */
+
+void brel(buf)
+void *buf;
+{
+    struct bfhead *b, *bn;
+
+#ifdef PIO_USE_MALLOC
+    //    printf("bget free %d %x\n",__LINE__,buf);
+    free(buf);
+    return;
+#endif
+
+
+    if(buf==NULL) return;       /* allow for null buffer */
+
+    b = BFH(((char *) buf) - sizeof(struct bhead));
+#ifdef BufStats
+    numrel++;                         /* Increment number of brel() calls */
+#endif
+    assert(buf != NULL);
+
+#ifdef BECtl
+    if (b->bh.bsize == 0) {           /* Directly-acquired buffer? */
+        struct bdhead *bdh;
+
+        bdh = BDH(((char *) buf) - sizeof(struct bdhead));
+        assert(b->bh.prevfree == 0);
+#ifdef BufStats
+        totalloc -= bdh->tsize;
+        assert(totalloc >= 0);
+        numdrel++;                    /* Number of direct releases */
+#endif /* BufStats */
+#ifdef FreeWipe
+        V memset((char *) buf, 0x55,
+                 (MemSize) (bdh->tsize - sizeof(struct bdhead)));
+#endif /* FreeWipe */
+        assert(relfcn != NULL);
+        (*relfcn)((void *) bdh);      /* Release it directly. */
+        return;
+    }
+#endif /* BECtl */
+
+    /* Buffer size must be negative, indicating that the buffer is
+       allocated. */
+
+    if (b->bh.bsize >= 0) {
+        bn = NULL;
+    }
+    assert(b->bh.bsize < 0);
+
+    /*  Back pointer in next buffer must be zero, indicating the
+        same thing: */
+
+    assert(BH((char *) b - b->bh.bsize)->prevfree == 0);
+
+#ifdef BufStats
+    totalloc += b->bh.bsize;
+    assert(totalloc >= 0);
+#endif
+
+    /* If the back link is nonzero, the previous buffer is free.  */
+
+    if (b->bh.prevfree != 0) {
+
+        /* The previous buffer is free.  Consolidate this buffer  with  it
+           by  adding  the  length  of  this  buffer  to the previous free
+           buffer.  Note that we subtract the size  in  the  buffer  being
+           released,  since  it's  negative to indicate that the buffer is
+           allocated. */
+
+        register bufsize size = b->bh.bsize;
+
+        /* Make the previous buffer the one we're working on. */
+        assert(BH((char *) b - b->bh.prevfree)->bsize == b->bh.prevfree);
+        b = BFH(((char *) b) - b->bh.prevfree);
+        b->bh.bsize -= size;
+    } else {
+
+        /* The previous buffer isn't allocated.  Insert this buffer
+           on the free list as an isolated free block. */
+
+        assert(freelist.ql.blink->ql.flink == &freelist);
+        assert(freelist.ql.flink->ql.blink == &freelist);
+        b->ql.flink = &freelist;
+        b->ql.blink = freelist.ql.blink;
+        freelist.ql.blink = b;
+        b->ql.blink->ql.flink = b;
+        b->bh.bsize = -b->bh.bsize;
+    }
+
+    /* Now we look at the next buffer in memory, located by advancing from
+       the  start  of  this  buffer  by its size, to see if that buffer is
+       free.  If it is, we combine  this  buffer  with  the  next  one  in
+       memory, dechaining the second buffer from the free list. */
+
+    bn =  BFH(((char *) b) + b->bh.bsize);
+    if (bn->bh.bsize > 0) {
+
+        /* The buffer is free.  Remove it from the free list and add
+           its size to that of our buffer. */
+
+        assert(BH((char *) bn + bn->bh.bsize)->prevfree == bn->bh.bsize);
+        assert(bn->ql.blink->ql.flink == bn);
+        assert(bn->ql.flink->ql.blink == bn);
+        bn->ql.blink->ql.flink = bn->ql.flink;
+        bn->ql.flink->ql.blink = bn->ql.blink;
+        b->bh.bsize += bn->bh.bsize;
+
+        /* Finally,  advance  to   the  buffer  that   follows  the  newly
+           consolidated free block.  We must set its  backpointer  to  the
+           head  of  the  consolidated free block.  We know the next block
+           must be an allocated block because the process of recombination
+           guarantees  that  two  free  blocks will never be contiguous in
+           memory.  */
+
+        bn = BFH(((char *) b) + b->bh.bsize);
+    }
+#ifdef FreeWipe
+    V memset(((char *) b) + sizeof(struct bfhead), 0x55,
+             (MemSize) (b->bh.bsize - sizeof(struct bfhead)));
+#endif
+    assert(bn->bh.bsize < 0);
+
+    /* The next buffer is allocated.  Set the backpointer in it  to  point
+       to this buffer; the previous free buffer in memory. */
+
+    bn->bh.prevfree = b->bh.bsize;
+
+#ifdef BECtl
+
+    /*  If  a  block-release function is defined, and this free buffer
+        constitutes the entire block, release it.  Note that  pool_len
+        is  defined  in  such a way that the test will fail unless all
+        pool blocks are the same size.  */
+
+    if (relfcn != NULL &&
+        ((bufsize) b->bh.bsize) == (pool_len - sizeof(struct bhead))) {
+
+        assert(b->bh.prevfree == 0);
+        assert(BH((char *) b + b->bh.bsize)->bsize == ESent);
+        assert(BH((char *) b + b->bh.bsize)->prevfree == b->bh.bsize);
+        /*  Unlink the buffer from the free list  */
+        b->ql.blink->ql.flink = b->ql.flink;
+        b->ql.flink->ql.blink = b->ql.blink;
+        //      printf("%s %d calling direct release for %x\n",__FILE__,__LINE__,b);
+        (*relfcn)(b);
+        //      printf("%s %d completed direct release \n",__FILE__,__LINE__);
+#ifdef BufStats
+        numprel++;                    /* Nr of expansion block releases */
+        numpblk--;                    /* Total number of blocks */
+        assert(numpblk == numpget - numprel);
+#endif /* BufStats */
+    }
+#endif /* BECtl */
+}
+
+#ifdef BECtl
+
+/*  BECTL  --  Establish automatic pool expansion control  */
+
+void bectl(compact, acquire, release, pool_incr)
+    int (*compact) _((bufsize sizereq, int sequence));
+void *(*acquire) _((bufsize size));
+void (*release) _((void *buf));
+bufsize pool_incr;
+{
+    compfcn = compact;
+    acqfcn = acquire;
+    relfcn = release;
+    exp_incr = pool_incr;
+}
+#endif
+
+/*  BPOOL  --  Add a region of memory to the buffer pool.  */
+
+void bpool(buf, len)
+void *buf;
+bufsize len;
+{
+    struct bfhead *b = BFH(buf);
+    struct bhead *bn;
+
+#ifdef SizeQuant
+    len &= ~(SizeQuant - 1);
+#endif
+#ifdef BECtl
+    if (pool_len == 0) {
+        pool_len = len;
+    } else if (len != pool_len) {
+        pool_len = -1;
+    }
+#ifdef BufStats
+    numpget++;                        /* Number of block acquisitions */
+    numpblk++;                        /* Number of blocks total */
+    assert(numpblk == numpget - numprel);
+#endif /* BufStats */
+#endif /* BECtl */
+
+    /* Since the block is initially occupied by a single free  buffer,
+       it  had  better  not  be  (much) larger than the largest buffer
+       whose size we can store in bhead.bsize. */
+
+    assert(len - sizeof(struct bhead) <= -((bufsize) ESent + 1));
+
+    /* Clear  the  backpointer at  the start of the block to indicate that
+       there  is  no  free  block  prior  to  this   one.    That   blocks
+       recombination when the first block in memory is released. */
+
+    b->bh.prevfree = 0;
+
+    /* Chain the new block to the free list. */
+
+    assert(freelist.ql.blink->ql.flink == &freelist);
+    assert(freelist.ql.flink->ql.blink == &freelist);
+    b->ql.flink = &freelist;
+    b->ql.blink = freelist.ql.blink;
+    freelist.ql.blink = b;
+    b->ql.blink->ql.flink = b;
+
+    /* Create a dummy allocated buffer at the end of the pool.  This dummy
+       buffer is seen when a buffer at the end of the pool is released and
+       blocks  recombination  of  the last buffer with the dummy buffer at
+       the end.  The length in the dummy buffer  is  set  to  the  largest
+       negative  number  to  denote  the  end  of  the pool for diagnostic
+       routines (this specific value is  not  counted  on  by  the  actual
+       allocation and release functions). */
+
+    len -= sizeof(struct bhead);
+    b->bh.bsize = (bufsize) len;
+#ifdef FreeWipe
+    V memset(((char *) b) + sizeof(struct bfhead), 0x55,
+             (MemSize) (len - sizeof(struct bfhead)));
+#endif
+    bn = BH(((char *) b) + len);
+    bn->prevfree = (bufsize) len;
+    /* Definition of ESent assumes two's complement! */
+    assert((~0) == -1);
+    bn->bsize = ESent;
+}
+
+#ifdef BufStats
+
+void bfreespace(bufsize *totfree, bufsize *maxfree)
+{
+    struct bfhead *b = freelist.ql.flink;
+    *totfree = 0;
+    *maxfree = -1;
+    while (b != &freelist) {
+        assert(b->bh.bsize > 0);
+        *totfree += b->bh.bsize;
+        if (b->bh.bsize > *maxfree) {
+            *maxfree = b->bh.bsize;
+        }
+        b = b->ql.flink;              /* Link to next buffer */
+    }
+}
+
+/*  BSTATS  --  Return buffer allocation free space statistics.  */
+
+void bstats(curalloc, totfree, maxfree, nget, nrel)
+    bufsize *curalloc, *totfree, *maxfree;
+long *nget, *nrel;
+{
+    *nget = numget;
+    *nrel = numrel;
+    *curalloc = totalloc;
+    bfreespace(totfree, maxfree);
+}
+
+#ifdef BECtl
+
+/*  BSTATSE  --  Return extended statistics  */
+
+void bstatse(pool_incr, npool, npget, nprel, ndget, ndrel)
+bufsize *pool_incr;
+long *npool, *npget, *nprel, *ndget, *ndrel;
+{
+    *pool_incr = (pool_len < 0) ? -exp_incr : exp_incr;
+    *npool = numpblk;
+    *npget = numpget;
+    *nprel = numprel;
+    *ndget = numdget;
+    *ndrel = numdrel;
+}
+#endif /* BECtl */
+#endif /* BufStats */
+
+#ifdef DumpData
+
+/*  BUFDUMP  --  Dump the data in a buffer.  This is called with the  user
+    data pointer, and backs up to the buffer header.  It will
+    dump either a free block or an allocated one.       */
+
+void bufdump(buf)
+void *buf;
+{
+    struct bfhead *b;
+    unsigned char *bdump;
+    bufsize bdlen;
+
+    b = BFH(((char *) buf) - sizeof(struct bhead));
+    assert(b->bh.bsize != 0);
+    if (b->bh.bsize < 0) {
+        bdump = (unsigned char *) buf;
+        bdlen = (-b->bh.bsize) - sizeof(struct bhead);
+    } else {
+        bdump = (unsigned char *) (((char *) b) + sizeof(struct bfhead));
+        bdlen = b->bh.bsize - sizeof(struct bfhead);
+    }
+
+    while (bdlen > 0) {
+        int i, dupes = 0;
+        bufsize l = bdlen;
+        char bhex[50], bascii[20];
+
+        if (l > 16) {
+            l = 16;
+        }
+
+        for (i = 0; i < l; i++) {
+            V sprintf(bhex + i * 3, "%02X ", bdump[i]);
+            bascii[i] = isprint(bdump[i]) ? bdump[i] : ' ';
+        }
+        bascii[i] = 0;
+        V printf("%-48s   %s\n", bhex, bascii);
+        bdump += l;
+        bdlen -= l;
+        while ((bdlen > 16) && (memcmp((char *) (bdump - 16),
+                                       (char *) bdump, 16) == 0)) {
+            dupes++;
+            bdump += 16;
+            bdlen -= 16;
+        }
+        if (dupes > 1) {
+            V printf(
+                "     (%d lines [%d bytes] identical to above line skipped)\n",
+                dupes, dupes * 16);
+        } else if (dupes == 1) {
+            bdump -= 16;
+            bdlen += 16;
+        }
+    }
+}
+#endif
+
+#ifdef BufDump
+
+/*  BPOOLD  --  Dump a buffer pool.  The buffer headers are always listed.
+    If DUMPALLOC is nonzero, the contents of allocated buffers
+    are  dumped.   If  DUMPFREE  is  nonzero,  free blocks are
+    dumped as well.  If FreeWipe  checking      is  enabled,  free
+    blocks      which  have  been clobbered will always be dumped. */
+
+void bpoold(buf, dumpalloc, dumpfree)
+void *buf;
+int dumpalloc, dumpfree;
+{
+    struct bfhead *b = BFH(buf);
+
+    while (b->bh.bsize != ESent) {
+        bufsize bs = b->bh.bsize;
+
+        if (bs < 0) {
+            bs = -bs;
+            V printf("Allocated buffer: size %6ld bytes.\n", (long) bs);
+            if (dumpalloc) {
+                bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+            }
+        } else {
+            char *lerr = "";
+
+            assert(bs > 0);
+            if ((b->ql.blink->ql.flink != b) ||
+                (b->ql.flink->ql.blink != b)) {
+                lerr = "  (Bad free list links)";
+            }
+            V printf("Free block:       size %6ld bytes.%s\n",
+                     (long) bs, lerr);
+#ifdef FreeWipe
+            lerr = ((char *) b) + sizeof(struct bfhead);
+            if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) ||
+                                                 (memcmp(lerr, lerr + 1,
+                                                         (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) {
+                V printf(
+                    "(Contents of above free block have been overstored.)\n");
+                bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+            } else
+#endif
+                if (dumpfree) {
+                    bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+                }
+        }
+        b = BFH(((char *) b) + bs);
+    }
+}
+#endif /* BufDump */
+
+#ifdef BufValid
+
+/*  BPOOLV  --  Validate a buffer pool.  If NDEBUG isn't defined,
+    any error generates an assertion failure.  */
+
+int bpoolv(buf)
+void *buf;
+{
+    struct bfhead *b = BFH(buf);
+
+    while (b->bh.bsize != ESent) {
+        bufsize bs = b->bh.bsize;
+
+        if (bs < 0) {
+            bs = -bs;
+        } else {
+            char *lerr = "";
+
+            assert(bs > 0);
+            if (bs <= 0) {
+                return 0;
+            }
+            if ((b->ql.blink->ql.flink != b) ||
+                (b->ql.flink->ql.blink != b)) {
+                V printf("Free block: size %6ld bytes.  (Bad free list links)\n",
+                         (long) bs);
+                assert(0);
+                return 0;
+            }
+#ifdef FreeWipe
+            lerr = ((char *) b) + sizeof(struct bfhead);
+            if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) ||
+                                                 (memcmp(lerr, lerr + 1,
+                                                         (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) {
+                V printf(
+                    "(Contents of above free block have been overstored.)\n");
+                bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+                assert(0);
+                return 0;
+            }
+#endif
+        }
+        b = BFH(((char *) b) + bs);
+    }
+    return 1;
+}
+#endif /* BufValid */
+
+/***********************\
+ *                      *
+ * Built-in test program *
+ *                      *
+        \***********************/
+
+#ifdef TestProg
+
+#define Repeatable  1                 /* Repeatable pseudorandom sequence */
+                                      /* If Repeatable is not defined, a
+                                         time-seeded pseudorandom sequence
+                                         is generated, exercising BGET with
+                                         a different pattern of calls on each
+                                         run. */
+#define OUR_RAND                      /* Use our own built-in version of
+                                         rand() to guarantee the test is
+                                         100% repeatable. */
+
+#ifdef BECtl
+#define PoolSize    300000            /* Test buffer pool size */
+#else
+#define PoolSize    50000             /* Test buffer pool size */
+#endif
+#define ExpIncr     32768             /* Test expansion block size */
+#define CompactTries 10               /* Maximum tries at compacting */
+
+#define dumpAlloc   0                 /* Dump allocated buffers ? */
+#define dumpFree    0                 /* Dump free buffers ? */
+
+#ifndef Repeatable
+extern long time();
+#endif
+
+extern char *malloc();
+extern int free _((char *));
+
+static char *bchain = NULL;           /* Our private buffer chain */
+static char *bp = NULL;               /* Our initial buffer pool */
+
+#include 
+
+#ifdef OUR_RAND
+
+static unsigned long int next = 1;
+
+/* Return next random integer */
+
+int rand()
+{
+    next = next * 1103515245L + 12345;
+    return (unsigned int) (next / 65536L) % 32768L;
+}
+
+/* Set seed for random generator */
+
+void srand(seed)
+unsigned int seed;
+{
+    next = seed;
+}
+#endif
+
+/*  STATS  --  Edit statistics returned by bstats() or bstatse().  */
+
+static void stats(when)
+char *when;
+{
+    bufsize cural, totfree, maxfree;
+    long nget, nfree;
+#ifdef BECtl
+    bufsize pincr;
+    long totblocks, npget, nprel, ndget, ndrel;
+#endif
+
+    bstats(&cural, &totfree, &maxfree, &nget, &nfree);
+    V printf(
+        "%s: %ld gets, %ld releases.  %ld in use, %ld free, largest = %ld\n",
+        when, nget, nfree, (long) cural, (long) totfree, (long) maxfree);
+#ifdef BECtl
+    bstatse(&pincr, &totblocks, &npget, &nprel, &ndget, &ndrel);
+    V printf(
+        "  Blocks: size = %ld, %ld (%ld bytes) in use, %ld gets, %ld frees\n",
+        (long)pincr, totblocks, pincr * totblocks, npget, nprel);
+    V printf("  %ld direct gets, %ld direct frees\n", ndget, ndrel);
+#endif /* BECtl */
+}
+
+#ifdef BECtl
+static int protect = 0;               /* Disable compaction during bgetr() */
+
+/*  BCOMPACT  --  Compaction call-back function.  */
+
+static int bcompact(bsize, seq)
+bufsize bsize;
+int seq;
+{
+#ifdef CompactTries
+    char *bc = bchain;
+    int i = rand() & 0x3;
+
+#ifdef COMPACTRACE
+    V printf("Compaction requested.  %ld bytes needed, sequence %d.\n",
+             (long) bsize, seq);
+#endif
+
+    if (protect || (seq > CompactTries)) {
+#ifdef COMPACTRACE
+        V printf("Compaction gave up.\n");
+#endif
+        return 0;
+    }
+
+    /* Based on a random cast, release a random buffer in the list
+       of allocated buffers. */
+
+    while (i > 0 && bc != NULL) {
+        bc = *((char **) bc);
+        i--;
+    }
+    if (bc != NULL) {
+        char *fb;
+
+        fb = *((char **) bc);
+        if (fb != NULL) {
+            *((char **) bc) = *((char **) fb);
+            brel((void *) fb);
+            return 1;
+        }
+    }
+
+#ifdef COMPACTRACE
+    V printf("Compaction bailed out.\n");
+#endif
+#endif /* CompactTries */
+    return 0;
+}
+
+/*  BEXPAND  --  Expand pool call-back function.  */
+
+static void *bexpand(size)
+bufsize size;
+{
+    void *np = NULL;
+    bufsize cural, totfree, maxfree;
+    long nget, nfree;
+
+    /* Don't expand beyond the total allocated size given by PoolSize. */
+
+    bstats(&cural, &totfree, &maxfree, &nget, &nfree);
+
+    if (cural < PoolSize) {
+        np = (void *) malloc((unsigned) size);
+    }
+#ifdef EXPTRACE
+    V printf("Expand pool by %ld -- %s.\n", (long) size,
+             np == NULL ? "failed" : "succeeded");
+#endif
+    return np;
+}
+
+/*  BSHRINK  --  Shrink buffer pool call-back function.  */
+
+static void bshrink(buf)
+void *buf;
+{
+    if (((char *) buf) == bp) {
+#ifdef EXPTRACE
+        V printf("Initial pool released.\n");
+#endif
+        bp = NULL;
+    }
+#ifdef EXPTRACE
+    V printf("Shrink pool.\n");
+#endif
+    free((char *) buf);
+}
+
+#endif /* BECtl */
+
+/*  Restrict buffer requests to those large enough to contain our pointer and
+    small enough for the CPU architecture.  */
+
+static bufsize blimit(bs)
+bufsize bs;
+{
+    if (bs < sizeof(char *)) {
+        bs = sizeof(char *);
+    }
+
+    /* This is written out in this ugly fashion because the
+       cool expression in sizeof(int) that auto-configured
+       to any length int befuddled some compilers. */
+
+    if (sizeof(int) == 2) {
+        if (bs > 32767) {
+            bs = 32767;
+        }
+    } else {
+        if (bs > 200000) {
+            bs = 200000;
+        }
+    }
+    return bs;
+}
+
+int main()
+{
+    int i;
+    double x;
+
+    /* Seed the random number generator.  If Repeatable is defined, we
+       always use the same seed.  Otherwise, we seed from the clock to
+       shake things up from run to run. */
+
+#ifdef Repeatable
+    V srand(1234);
+#else
+    V srand((int) time((long *) NULL));
+#endif
+
+    /*  Compute x such that pow(x, p) ranges between 1 and 4*ExpIncr as
+        p ranges from 0 to ExpIncr-1, with a concentration in the lower
+        numbers.  */
+
+    x = 4.0 * ExpIncr;
+    x = log(x);
+    x = exp(log(4.0 * ExpIncr) / (ExpIncr - 1.0));
+
+#ifdef BECtl
+    bectl(bcompact, bexpand, bshrink, (bufsize) ExpIncr);
+    bp = malloc(ExpIncr);
+    assert(bp != NULL);
+    bpool((void *) bp, (bufsize) ExpIncr);
+#else
+    bp = malloc(PoolSize);
+    assert(bp != NULL);
+    bpool((void *) bp, (bufsize) PoolSize);
+#endif
+
+    stats("Create pool");
+    V bpoolv((void *) bp);
+    bpoold((void *) bp, dumpAlloc, dumpFree);
+
+    for (i = 0; i < TestProg; i++) {
+        char *cb;
+        bufsize bs = pow(x, (double) (rand() & (ExpIncr - 1)));
+
+        assert(bs <= (((bufsize) 4) * ExpIncr));
+        bs = blimit(bs);
+        if (rand() & 0x400) {
+            cb = (char *) bgetz(bs);
+        } else {
+            cb = (char *) bget(bs);
+        }
+        if (cb == NULL) {
+#ifdef EasyOut
+            break;
+#else
+            char *bc = bchain;
+
+            if (bc != NULL) {
+                char *fb;
+
+                fb = *((char **) bc);
+                if (fb != NULL) {
+                    *((char **) bc) = *((char **) fb);
+                    brel((void *) fb);
+                }
+                continue;
+            }
+#endif
+        }
+        *((char **) cb) = (char *) bchain;
+        bchain = cb;
+
+        /* Based on a random cast, release a random buffer in the list
+           of allocated buffers. */
+
+        if ((rand() & 0x10) == 0) {
+            char *bc = bchain;
+            int i = rand() & 0x3;
+
+            while (i > 0 && bc != NULL) {
+                bc = *((char **) bc);
+                i--;
+            }
+            if (bc != NULL) {
+                char *fb;
+
+                fb = *((char **) bc);
+                if (fb != NULL) {
+                    *((char **) bc) = *((char **) fb);
+                    brel((void *) fb);
+                }
+            }
+        }
+
+        /* Based on a random cast, reallocate a random buffer in the list
+           to a random size */
+
+        if ((rand() & 0x20) == 0) {
+            char *bc = bchain;
+            int i = rand() & 0x3;
+
+            while (i > 0 && bc != NULL) {
+                bc = *((char **) bc);
+                i--;
+            }
+            if (bc != NULL) {
+                char *fb;
+
+                fb = *((char **) bc);
+                if (fb != NULL) {
+                    char *newb;
+
+                    bs = pow(x, (double) (rand() & (ExpIncr - 1)));
+                    bs = blimit(bs);
+#ifdef BECtl
+                    protect = 1;      /* Protect against compaction */
+#endif
+                    newb = (char *) bgetr((void *) fb, bs);
+#ifdef BECtl
+                    protect = 0;
+#endif
+                    if (newb != NULL) {
+                        *((char **) bc) = newb;
+                    }
+                }
+            }
+        }
+    }
+    stats("\nAfter allocation");
+    if (bp != NULL) {
+        V bpoolv((void *) bp);
+        bpoold((void *) bp, dumpAlloc, dumpFree);
+    }
+
+    while (bchain != NULL) {
+        char *buf = bchain;
+
+        bchain = *((char **) buf);
+        brel((void *) buf);
+    }
+    stats("\nAfter release");
+#ifndef BECtl
+    if (bp != NULL) {
+        V bpoolv((void *) bp);
+        bpoold((void *) bp, dumpAlloc, dumpFree);
+    }
+#endif
+
+    return 0;
+}
+#endif
diff --git a/src/externals/pio2/src/clib/bget.h b/src/externals/pio2/src/clib/bget.h
new file mode 100644
index 00000000000..2b5be7dcdc0
--- /dev/null
+++ b/src/externals/pio2/src/clib/bget.h
@@ -0,0 +1,41 @@
+/*
+
+  Interface definitions for bget.c, the memory management package.
+
+*/
+#ifndef _BGET_H_
+#define _BGET_H_
+
+/* in PIO we use DEBUG bget uses NDEBUG */
+//#ifndef DEBUG
+//#undef NDEBUG
+//#endif
+
+#ifndef _
+#ifdef PROTOTYPES
+#define  _(x)  x                      /* If compiler knows prototypes */
+#else
+#define  _(x)  ()                     /* It it doesn't */
+#endif /* PROTOTYPES */
+#endif
+
+typedef long bufsize;
+void    bpool       _((void *buffer, bufsize len));
+void   *bget        _((bufsize size));
+void   *bgetz       _((bufsize size));
+void   *bgetr       _((void *buffer, bufsize newsize));
+void    brel        _((void *buf));
+void    bectl       _((int (*compact)(bufsize sizereq, int sequence),
+                       void *(*acquire)(bufsize size),
+                       void (*release)(void *buf), bufsize pool_incr));
+void    bstats      _((bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
+                       long *nget, long *nrel));
+void    bstatse     _((bufsize *pool_incr, long *npool, long *npget,
+                       long *nprel, long *ndget, long *ndrel));
+void    bufdump     _((void *buf));
+void    bpoold      _((void *pool, int dumpalloc, int dumpfree));
+int     bpoolv      _((void *pool));
+void bpoolrelease  _();
+void bfreespace  _((bufsize *maxfree, bufsize *totfree));
+
+#endif
diff --git a/externals/pio2/src/clib/config.h.in b/src/externals/pio2/src/clib/config.h.in
similarity index 84%
rename from externals/pio2/src/clib/config.h.in
rename to src/externals/pio2/src/clib/config.h.in
index e4be61d9e0c..1722872c305 100644
--- a/externals/pio2/src/clib/config.h.in
+++ b/src/externals/pio2/src/clib/config.h.in
@@ -19,4 +19,7 @@
  * will use the included bget() package for memory management. */
 #define PIO_USE_MALLOC @USE_MALLOC@
 
+/** Set to non-zero to turn on logging. Output may be large. */
+#define PIO_ENABLE_LOGGING @ENABLE_LOGGING@
+
 #endif /* _PIO_CONFIG_ */
diff --git a/externals/pio2/src/clib/dtypes.h b/src/externals/pio2/src/clib/dtypes.h
similarity index 100%
rename from externals/pio2/src/clib/dtypes.h
rename to src/externals/pio2/src/clib/dtypes.h
diff --git a/externals/pio2/src/clib/ncparser.pl b/src/externals/pio2/src/clib/ncparser.pl
similarity index 100%
rename from externals/pio2/src/clib/ncparser.pl
rename to src/externals/pio2/src/clib/ncparser.pl
diff --git a/externals/pio2/src/clib/ncputgetparser.pl b/src/externals/pio2/src/clib/ncputgetparser.pl
similarity index 100%
rename from externals/pio2/src/clib/ncputgetparser.pl
rename to src/externals/pio2/src/clib/ncputgetparser.pl
diff --git a/src/externals/pio2/src/clib/pio.h b/src/externals/pio2/src/clib/pio.h
new file mode 100644
index 00000000000..fe234a15593
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio.h
@@ -0,0 +1,1142 @@
+/**
+ * @file
+ * Public headers for the PIO C interface.
+ * @author Jim Edwards
+ * @date  2014
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#ifndef _PIO_H_
+#define _PIO_H_
+#include 
+#include 
+#include 
+#include  /* memcpy */
+#include 
+
+#ifdef _NETCDF
+#include 
+#ifdef _NETCDF4
+#include 
+#endif
+#endif
+#ifdef _PNETCDF
+#include 
+#endif
+
+#ifndef MPI_OFFSET
+/** MPI_OFFSET is an integer type of size sufficient to represent the
+ * size (in bytes) of the largest file supported by MPI. In some MPI
+ * implementations MPI_OFFSET is not properly defined.  */
+#define MPI_OFFSET  MPI_LONG_LONG
+#endif
+
+/** PIO_OFFSET is an integer type of size sufficient to represent the
+ * size (in bytes) of the largest file supported by MPI. */
+#define PIO_OFFSET MPI_OFFSET
+#define PIO_Offset MPI_Offset
+
+/** The maximum number of variables allowed in a netCDF file. */
+#define PIO_MAX_VARS NC_MAX_VARS
+
+/** Pass this to PIOc_set_iosystem_error_handling() as the iosysid in
+ * order to set default error handling. */
+#define PIO_DEFAULT (-1)
+
+/** Used in the decomposition netCDF file. */
+
+/* Holds the version of the decomposition file. */
+#define DECOMP_VERSION_ATT_NAME "version"
+
+/* Holds the maximum length of any task map. */
+#define DECOMP_MAX_MAPLEN_ATT_NAME "max_maplen"
+
+/* Name of title attribute. */
+#define DECOMP_TITLE_ATT_NAME "title"
+
+/* Name of history attribute. */
+#define DECOMP_HISTORY_ATT_NAME "history"
+
+/* Name of source attribute. */
+#define DECOMP_SOURCE_ATT_NAME "source"
+
+/* Name of array order (C or Fortran) attribute. */
+#define DECOMP_ORDER_ATT_NAME "array_order"
+
+/* Name of backtrace attribute. */
+#define DECOMP_BACKTRACE_ATT_NAME "backtrace"
+
+/* Name for the dim dim in decomp file. */
+#define DECOMP_DIM_DIM "dims"
+
+/* Name for the npes dim in decomp file. */
+#define DECOMP_TASK_DIM_NAME "task"
+
+/* Name for the npes dim in decomp file. */
+#define DECOMP_MAPELEM_DIM_NAME "map_element"
+
+#define DECOMP_NDIMS "ndims"
+
+/* Name of var in decomp file that holds global array sizes. */
+#define DECOMP_GLOBAL_SIZE_VAR_NAME "global_size"
+
+/* Name of var in decomp file that holds the length of the map for
+ * each task. */
+#define DECOMP_MAPLEN_VAR_NAME "maplen"
+
+/* Name of var in decomp file that holds map. */
+#define DECOMP_MAP_VAR_NAME "map"
+
+/* String used to indicate a decomposition file is in C
+ * array-order. */
+#define DECOMP_C_ORDER_STR "C"
+
+/* String used to indicate a decomposition file is in Fortran
+ * array-order. */
+#define DECOMP_FORTRAN_ORDER_STR "Fortran"
+
+
+/**
+ * Variable description structure.
+ */
+typedef struct var_desc_t
+{
+    /** The unlimited dimension in the netCDF file (typically the time
+     * dimension). -1 if there is no unlimited dimension. */
+    int record;
+
+    /** Number of dimensions for this variable. */
+    int ndims;
+
+    /** ID of each outstanding pnetcdf request for this variable. */
+    int *request;
+
+    /** Number of requests bending with pnetcdf. */
+    int nreqs;
+
+    /** Buffer that contains the fill value for this variable. */
+    void *fillbuf;
+
+    /** Data buffer for this variable. */
+    void *iobuf;
+} var_desc_t;
+
+/**
+ * IO region structure.
+ *
+ * Each IO region is a unit of data which can be described using start
+ * and count arrays. Each IO task may in general have multiple io
+ * regions per variable.  The box rearranger will have at most one io
+ * region per variable.
+ *
+ * The write from a particular IO task is divided into 1 or more
+ * regions each of which can be described using start and count. The
+ * io_region typedef is a linked list of those regions.
+ */
+typedef struct io_region
+{
+    /** The offset from the beginning of the data buffer to the
+     * beginning of this region.  */
+    int loffset;
+
+    /** Start array for this region. */
+    PIO_Offset *start;
+
+    /** Count array for this region. */
+    PIO_Offset *count;
+
+    /** Pointer to the next io_region in the list. */
+    struct io_region *next;
+} io_region;
+
+/**
+ * Rearranger comm type. The rearranger option values must match the
+ * definitions in the fortran interface.
+ */
+enum PIO_REARR_COMM_TYPE
+{
+    /** Point to point */
+    PIO_REARR_COMM_P2P = (0),
+
+    /** Collective */
+    PIO_REARR_COMM_COLL
+};
+
+/**
+ * Rearranger comm flow control direction. The rearranger option
+ * values must match the definitions in the fortran interface.
+ */
+enum PIO_REARR_COMM_FC_DIR
+{
+    /** Comp procs to io procs and vice versa */
+    PIO_REARR_COMM_FC_2D_ENABLE = (0),
+
+    /** Comp procs to io procs only */
+    PIO_REARR_COMM_FC_1D_COMP2IO,
+
+    /** IO procs to comp procs only */
+    PIO_REARR_COMM_FC_1D_IO2COMP,
+
+    /** Disable flow control */
+    PIO_REARR_COMM_FC_2D_DISABLE
+};
+
+/* Constant to indicate unlimited requests. */
+#define PIO_REARR_COMM_UNLIMITED_PEND_REQ -1
+
+/**
+ * Rearranger comm flow control options.
+ */
+typedef struct rearr_comm_fc_opt
+{
+    /** Enable handshake */
+    bool enable_hs;
+
+    /** Enable isends - if false use blocking sends */
+    bool enable_isend;
+
+    /** Max pending requests
+     * (PIO_REARR_COMM_UNLIMITED_PEND_REQ => unlimited pend req).
+     * This is the number of messages allowed to be in flight at one
+     * time. On some systems posting all messages at once creates a
+     * significant bottleneck in communications and throttling in this
+     * manner improves overall performance. */
+    int max_pend_req;
+} rearr_comm_fc_opt_t;
+
+/**
+ * Rearranger options
+ */
+typedef struct rearr_opt
+{
+    /** Comm type - see PIO_REARR_COMM_TYPE */
+    int comm_type;
+
+    /** Comm flow control dir - see PIO_REARR_COMM_FC_DIR */
+    int fcd;
+
+    /** flow control opts, comp to io procs */
+    rearr_comm_fc_opt_t comm_fc_opts_comp2io;
+
+    /** flow control opts, io to comp procs */
+    rearr_comm_fc_opt_t comm_fc_opts_io2comp;
+} rearr_opt_t;
+
+/**
+ * IO descriptor structure.
+ *
+ * This structure defines the mapping for a given variable between
+ * compute and IO decomposition.
+ */
+typedef struct io_desc_t
+{
+    /** The ID of this io_desc_t. */
+    int ioid;
+
+    /** The length of the decomposition map. */
+    int maplen;
+
+    /** A 1-D array with iodesc->maplen elements, which are the
+     * 1-based mappings to the global array for that task. */
+    PIO_Offset *map;
+
+    /** Number of tasks involved in the communication between comp and
+     * io tasks. */
+    int nrecvs;
+
+    /** Local size of the decomposition array on the compute node. On
+        each compute task the application passes a compmap array of
+        length ndof. This array describes the arrangement of data in
+        memory on that task. */
+    int ndof;
+
+    /** All vars included in this io_desc_t have the same number of
+     * dimensions. */
+    int ndims;
+
+    /** An array of size ndims with the length of each dimension. */
+    int *dimlen;
+
+    /** The actual number of IO tasks participating. */
+    int num_aiotasks;
+
+    /** The rearranger in use for this variable. */
+    int rearranger;
+
+    /** Maximum number of regions in the decomposition. */
+    int maxregions;
+
+    /** Does this decomp leave holes in the field (true) or write
+     * everywhere (false) */
+    bool needsfill;
+
+    /** The maximum number of bytes of this iodesc before flushing. */
+    int maxbytes;
+
+    /** The MPI type of the data. */
+    MPI_Datatype basetype;
+
+    /** Length of the iobuffer on this task for a single field on the
+     * IO node. The arrays from compute nodes gathered and rearranged
+     * to the io-nodes (which are sometimes collocated with compute
+     * nodes), each io task contains data from the compmap of one or
+     * more compute tasks in the iomap array. */
+    PIO_Offset llen;
+
+    /** Maximum llen participating. */
+    int maxiobuflen;
+
+    /** Array of tasks received from in pio_swapm(). */
+    int *rfrom;
+
+    /** Array of counts of data to be received from each task in
+     * pio_swapm(). */
+    int *rcount;
+
+    /** Array of data count to send to each task in the communication
+     * in pio_swapm(). */
+    int *scount;
+
+    /** Send index. */
+    PIO_Offset *sindex;
+
+    /** Receive index. */
+    PIO_Offset *rindex;
+
+    /** Array of receive MPI types in pio_swapm() call. */
+    MPI_Datatype *rtype;
+
+    /** Array of send MPI types in pio_swapm() call. */
+    MPI_Datatype *stype;
+
+    /** Number of send MPI types in pio_swapm() call. */
+    int num_stypes;
+
+    /** Used when writing fill data. */
+    int holegridsize;
+
+    /** max holegridsize across all io tasks, needed for netcdf and netcdf4c serial */
+    int maxholegridsize;
+
+    /** Used when writing fill data. */
+    int maxfillregions;
+
+    /** Linked list of regions. */
+    io_region *firstregion;
+
+    /** Used when writing fill data. */
+    io_region *fillregion;
+
+    /** Rearranger flow control options
+     *  (handshake, non-blocking sends, pending requests)
+     */
+    rearr_opt_t rearr_opts;
+
+    /** In the subset communicator each io task is associated with a
+     * unique group of comp tasks this is the communicator for that
+     * group. */
+    MPI_Comm subset_comm;
+
+    /** Pointer to the next io_desc_t in the list. */
+    struct io_desc_t *next;
+} io_desc_t;
+
+/**
+ * IO system descriptor structure.
+ *
+ * This structure contains the general IO subsystem data and MPI
+ * structure
+ */
+typedef struct iosystem_desc_t
+{
+    /** The ID of this iosystem_desc_t. This will be obtained by
+     * calling PIOc_Init_Intercomm() or PIOc_Init_Intracomm(). */
+    int iosysid;
+
+    /** This is an MPI intra communicator that includes all the tasks in
+     * both the IO and the computation communicators. */
+    MPI_Comm union_comm;
+
+    /** This is an MPI intra communicator that includes all the tasks
+     * involved in IO. */
+    MPI_Comm io_comm;
+
+    /** This is an MPI intra communicator that includes all the tasks
+     * involved in computation. */
+    MPI_Comm comp_comm;
+
+    /** This is an MPI inter communicator between IO communicator and
+     * computation communicator. */
+    MPI_Comm intercomm;
+
+    /** This is a copy (but not an MPI copy) of either the comp (for
+     * non-async) or the union (for async) communicator. */
+    MPI_Comm my_comm;
+
+    /** This MPI group contains the processors involved in
+     * computation. */
+    MPI_Group compgroup;
+
+    /** This MPI group contains the processors involved in I/O. */
+    MPI_Group iogroup;
+
+    /** The number of tasks in the IO communicator. */
+    int num_iotasks;
+
+    /** The number of tasks in the computation communicator. */
+    int num_comptasks;
+
+    /** Rank of this task in the union communicator. */
+    int union_rank;
+
+    /** The rank of this process in the computation communicator, or -1
+     * if this process is not part of the computation communicator. */
+    int comp_rank;
+
+    /** The rank of this process in the IO communicator, or -1 if this
+     * process is not part of the IO communicator. */
+    int io_rank;
+
+    /** Set to MPI_ROOT if this task is the master of IO communicator, 0
+     * otherwise. */
+    int iomaster;
+
+    /** Set to MPI_ROOT if this task is the master of comp communicator, 0
+     * otherwise. */
+    int compmaster;
+
+    /** Rank of IO root task (which is rank 0 in io_comm) in the union
+     * communicator. Will always be 0 for async situations. */
+    int ioroot;
+
+    /** Rank of computation root task (which is rank 0 in
+     * comm_comms[cmp]) in the union communicator. Will always = number
+     * of IO tasks in async situations. */
+    int comproot;
+
+    /** An array of the ranks of all IO tasks within the union
+     * communicator. */
+    int *ioranks;
+
+    /** Controls handling errors. */
+    int error_handler;
+
+    /** The rearranger decides which parts of a distributed array are
+     * handled by which IO tasks. */
+    int default_rearranger;
+
+    /** True if asynchronous interface is in use. */
+    bool async_interface;
+
+    /** True if this task is a member of the IO communicator. */
+    bool ioproc;
+
+    /** MPI Info object. */
+    MPI_Info info;
+
+    /** Index of this component in the list of components. */
+    int comp_idx;
+
+    /** Rearranger options. */
+    rearr_opt_t rearr_opts;
+
+    /** Pointer to the next iosystem_desc_t in the list. */
+    struct iosystem_desc_t *next;
+} iosystem_desc_t;
+
+/**
+ * The multi buffer holds data from one or more variables. Data are
+ * accumulated in the multi-buffer.
+ */
+typedef struct wmulti_buffer
+{
+    /** The ID that describes the decomposition, as returned from
+     * PIOc_Init_Decomp().  */
+    int ioid;
+
+    /** Number of variables in this multi variable buffer. */
+    int validvars;
+
+    /** Size of this variables data on local task. All vars in the
+     * multi-buffer have the same size. */
+    int arraylen;
+
+    /** Array of varids. */
+    int *vid;
+
+    /** An array of current record numbers, for record vars. One
+     * element per variable. */
+    int *frame;
+
+    /** Array of fill values used for each var. */
+    void *fillvalue;
+
+    /** Pointer to the data. */
+    void *data;
+
+    /** Pointer to the next multi-buffer in the list. */
+    struct wmulti_buffer *next;
+} wmulti_buffer;
+
+/**
+ * File descriptor structure.
+ *
+ * This structure holds information associated with each open file
+ */
+typedef struct file_desc_t
+{
+    /** The IO system ID used to open this file. */
+    iosystem_desc_t *iosystem;
+
+    /** The ncid returned for this file by the underlying library
+     * (netcdf or pnetcdf). */
+    int fh;
+
+    /** The ncid that will be returned to the user. */
+    int pio_ncid;
+
+    /** The PIO_TYPE value that was used to open this file. */
+    int iotype;
+
+    /** List of variables in this file. */
+    struct var_desc_t varlist[PIO_MAX_VARS];
+
+    /** ??? */
+    int mode;
+
+    /** The wmulti_buffer is used to aggregate multiple variables with
+     * the same communication pattern prior to a write. */
+    struct wmulti_buffer buffer;
+
+    /** Pointer to the next file_desc_t in the list of open files. */
+    struct file_desc_t *next;
+
+    /** True if this task should participate in IO (only true for one
+     * task with netcdf serial files. */
+    int do_io;
+} file_desc_t;
+
+/**
+ * These are the supported methods of reading/writing netCDF
+ * files. (Not all methods can be used with all netCDF files.)
+ */
+enum PIO_IOTYPE
+{
+    /** Parallel Netcdf  (parallel) */
+    PIO_IOTYPE_PNETCDF = 1,
+
+    /** Netcdf3 Classic format (serial) */
+    PIO_IOTYPE_NETCDF = 2,
+
+    /**  NetCDF4 (HDF5) compressed format (serial) */
+    PIO_IOTYPE_NETCDF4C = 3,
+
+    /** NetCDF4 (HDF5) parallel */
+    PIO_IOTYPE_NETCDF4P = 4
+};
+
+/**
+ * These are the supported output data rearrangement methods.
+ */
+enum PIO_REARRANGERS
+{
+    /** Box rearranger. */
+    PIO_REARR_BOX = 1,
+
+    /** Subset rearranger. */
+    PIO_REARR_SUBSET = 2
+};
+
+/**
+ * These are the supported error handlers.
+ */
+enum PIO_ERROR_HANDLERS
+{
+    /** Errors cause abort. */
+    PIO_INTERNAL_ERROR = (-51),
+
+    /** Error codes are broadcast to all tasks. */
+    PIO_BCAST_ERROR = (-52),
+
+    /** Errors are returned to caller with no internal action. */
+    PIO_RETURN_ERROR = (-53)
+};
+
+#if defined( _PNETCDF) || defined(_NETCDF)
+
+#define PIO_GLOBAL NC_GLOBAL
+#define PIO_UNLIMITED NC_UNLIMITED
+
+/* NetCDF types. */
+#define PIO_BYTE   NC_BYTE
+#define PIO_CHAR   NC_CHAR
+#define PIO_SHORT  NC_SHORT
+#define PIO_INT    NC_INT
+#define PIO_FLOAT  NC_FLOAT
+#define PIO_REAL   NC_FLOAT
+#define PIO_DOUBLE NC_DOUBLE
+#define PIO_UBYTE  NC_UBYTE
+#define PIO_USHORT NC_USHORT
+#define PIO_UINT   NC_UINT
+#define PIO_INT64  NC_INT64
+#define PIO_UINT64 NC_UINT64
+#define PIO_STRING NC_STRING
+
+/* NetCDF flags. */
+#define PIO_WRITE  NC_WRITE
+#define PIO_NOWRITE  NC_NOWRITE
+#define PIO_CLOBBER NC_CLOBBER
+#define PIO_NOCLOBBER NC_NOCLOBBER
+#define PIO_FILL NC_FILL
+#define PIO_NOFILL NC_NOFILL
+#define PIO_MAX_NAME NC_MAX_NAME
+#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS
+#define PIO_64BIT_OFFSET NC_64BIT_OFFSET
+
+/** NC_64BIT_DATA This is a problem - need to define directly instead
+ * of using include file. */
+#define PIO_64BIT_DATA 0x0010
+
+/** Define the netCDF-based error codes. */
+#define PIO_NOERR  NC_NOERR
+#define PIO_EBADID NC_EBADID
+#define PIO_ENFILE NC_ENFILE
+#define PIO_EEXIST NC_EEXIST
+#define PIO_EINVAL NC_EINVAL
+#define PIO_EPERM NC_EPERM
+#define PIO_ENOTINDEFINE NC_ENOTINDEFINE
+#define PIO_EINDEFINE NC_EINDEFINE
+#define PIO_EINVALCOORDS NC_EINVALCOORDS
+#define PIO_EMAXDIMS NC_EMAXDIMS
+#define PIO_ENAMEINUSE NC_ENAMEINUSE
+#define PIO_ENOTATT NC_ENOTATT
+#define PIO_EMAXATTS NC_EMAXATTS
+#define PIO_EBADTYPE NC_EBADTYPE
+#define PIO_EBADDIM NC_EBADDIM
+#define PIO_EUNLIMPOS NC_EUNLIMPOS
+#define PIO_EMAXVARS NC_EMAXVARS
+#define PIO_ENOTVAR NC_ENOTVAR
+#define PIO_EGLOBAL NC_EGLOBAL
+#define PIO_ENOTNC NC_ENOTNC
+#define PIO_ESTS NC_ESTS
+#define PIO_EMAXNAME NC_EMAXNAME
+#define PIO_EUNLIMIT NC_EUNLIMIT
+#define PIO_ENORECVARS NC_ENORECVARS
+#define PIO_ECHAR NC_ECHAR
+#define PIO_EEDGE NC_EEDGE
+#define PIO_ESTRIDE NC_ESTRIDE
+#define PIO_EBADNAME NC_EBADNAME
+#define PIO_ERANGE NC_ERANGE
+#define PIO_ENOMEM NC_ENOMEM
+#define PIO_EVARSIZE NC_EVARSIZE
+#define PIO_EDIMSIZE NC_EDIMSIZE
+#define PIO_ETRUNC NC_ETRUNC
+#define PIO_EAXISTYPE NC_EAXISTYPE
+#define PIO_EDAP NC_EDAP
+#define PIO_ECURL NC_ECURL
+#define PIO_EIO NC_EIO
+#define PIO_ENODATA NC_ENODATA
+#define PIO_EDAPSVC NC_EDAPSVC
+#define PIO_EDAS NC_EDAS
+#define PIO_EDDS NC_EDDS
+#define PIO_EDATADDS NC_EDATADDS
+#define PIO_EDAPURL NC_EDAPURL
+#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT
+#define PIO_ETRANSLATION NC_ETRANSLATION
+#define PIO_EHDFERR NC_EHDFERR
+#define PIO_ECANTREAD NC_ECANTREAD
+#define PIO_ECANTWRITE NC_ECANTWRITE
+#define PIO_ECANTCREATE NC_ECANTCREATE
+#define PIO_EFILEMETA NC_EFILEMETA
+#define PIO_EDIMMETA NC_EDIMMETA
+#define PIO_EATTMETA NC_EATTMETA
+#define PIO_EVARMETA NC_EVARMETA
+#define PIO_ENOCOMPOUND NC_ENOCOMPOUND
+#define PIO_EATTEXISTS NC_EATTEXISTS
+#define PIO_ENOTNC4 NC_ENOTNC4
+#define PIO_ESTRICTNC3 NC_ESTRICTNC3
+#define PIO_ENOTNC3 NC_ENOTNC3
+#define PIO_ENOPAR NC_ENOPAR
+#define PIO_EPARINIT NC_EPARINIT
+#define PIO_EBADGRPID NC_EBADGRPID
+#define PIO_EBADTYPID NC_EBADTYPID
+#define PIO_ETYPDEFINED NC_ETYPDEFINED
+#define PIO_EBADFIELD NC_EBADFIELD
+#define PIO_EBADCLASS NC_EBADCLASS
+#define PIO_EMAPTYPE NC_EMAPTYPE
+#define PIO_ELATEFILL NC_ELATEFILL
+#define PIO_ELATEDEF NC_ELATEDEF
+#define PIO_EDIMSCALE NC_EDIMSCALE
+#define PIO_ENOGRP NC_ENOGRP
+#define PIO_ESTORAGE NC_ESTORAGE
+#define PIO_EBADCHUNK NC_EBADCHUNK
+#define PIO_ENOTBUILT NC_ENOTBUILT
+#define PIO_EDISKLESS NC_EDISKLESS
+#define PIO_FILL_DOUBLE NC_FILL_DOUBLE
+#define PIO_FILL_FLOAT NC_FILL_FLOAT
+#define PIO_FILL_INT NC_FILL_INT
+#define PIO_FILL_CHAR NC_FILL_CHAR
+#endif /*  defined( _PNETCDF) || defined(_NETCDF) */
+
+/** Define the extra error codes for the parallel-netcdf library. */
+#ifdef _PNETCDF
+#define PIO_EINDEP  NC_EINDEP
+#else  /* _PNETCDF */
+#define PIO_EINDEP  (-203)
+#endif /* _PNETCDF */
+
+/** Define error codes for PIO. */
+#define PIO_FIRST_ERROR_CODE (-500)
+#define PIO_EBADIOTYPE  (-500)
+
+/** ??? */
+#define PIO_REQ_NULL (NC_REQ_NULL-1)
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+    /* Error handling. */
+    int PIOc_strerror(int pioerr, char *errstr);
+    int PIOc_set_log_level(int level);
+
+    /* Decomposition. */
+
+    /* Init decomposition with 1-based compmap array. */
+    int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int maplen,
+                        const PIO_Offset *compmap, int *ioidp, const int *rearr,
+                        const PIO_Offset *iostart, const PIO_Offset *iocount);
+    int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims,
+                           const long int *start, const long int *count, int *ioidp);
+
+    /* Init decomposition with 0-based compmap array. */
+    int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int maplen,
+                         const PIO_Offset *compmap, int *ioidp, const int *rearranger,
+                         const PIO_Offset *iostart, const PIO_Offset *iocount);
+    
+    int PIOc_freedecomp(int iosysid, int ioid);
+    int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
+                     PIO_Offset **map, MPI_Comm comm);
+    int PIOc_readmap_from_f90(const char *file,int *ndims, int **gdims, PIO_Offset *maplen,
+                              PIO_Offset **map, int f90_comm);
+    int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
+                      PIO_Offset *map, MPI_Comm comm);
+    int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
+                               PIO_Offset maplen, const PIO_Offset *map, int f90_comm);
+
+    /* Write a decomposition file. */
+    int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm);
+
+    /* Write a decomposition file using netCDF. */
+    int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
+                             MPI_Comm comm, char *title, char *history, int fortran_order);
+
+    /* Read a netCDF decomposition file. */
+    int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioid, MPI_Comm comm,
+                            int pio_type, char *title, char *history, int *fortran_order);
+
+    /* Initializing IO system. */
+    int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count,
+                        int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm,
+                        int *iosysidp);
+    int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms,
+                            MPI_Comm io_comm, int *iosysidp);
+    int PIOc_get_numiotasks(int iosysid, int *numiotasks);
+    int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
+                            int *iosysidp);
+    int PIOc_finalize(int iosysid);
+
+    /* Set error handling for entire io system. */
+    int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method);
+
+    /* Set error handling for entire io system. */
+    int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method);
+
+    int PIOc_iam_iotask(int iosysid, bool *ioproc);
+    int PIOc_iotask_rank(int iosysid, int *iorank);
+    int PIOc_iosystem_is_active(int iosysid, bool *active);
+    int PIOc_iotype_available(int iotype);
+    int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd,
+                            bool enable_hs_c2i, bool enable_isend_c2i,
+                            int max_pend_req_c2i,
+                            bool enable_hs_i2c, bool enable_isend_i2c,
+                            int max_pend_req_i2c);
+    /* Distributed data. */
+    int PIOc_advanceframe(int ncid, int varid);
+    int PIOc_setframe(int ncid, int varid, int frame);
+    int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
+                          void *fillvalue);
+    int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen,
+                                void *array, const int *frame, void **fillvalue, bool flushtodisk);
+    int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array);
+    int PIOc_get_local_array_size(int ioid);
+
+    /* Handling files. */
+    int PIOc_redef(int ncid);
+    int PIOc_enddef(int ncid);
+    int PIOc_sync(int ncid);
+    int PIOc_deletefile(int iosysid, const char *filename);
+    int PIOc_createfile(int iosysid, int *ncidp,  int *iotype, const char *fname, int mode);
+    int PIOc_create(int iosysid, const char *path, int cmode, int *ncidp);
+    int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *fname, int mode);
+    int PIOc_open(int iosysid, const char *path, int mode, int *ncidp);
+    int PIOc_closefile(int ncid);
+    int PIOc_inq_format(int ncid, int *formatp);
+    int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp);
+    int PIOc_inq_ndims(int ncid, int *ndimsp);
+    int PIOc_inq_nvars(int ncid, int *nvarsp);
+    int PIOc_inq_natts(int ncid, int *ngattsp);
+    int PIOc_inq_unlimdim(int ncid, int *unlimdimidp);
+    int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep);
+    int PIOc_set_blocksize(int newblocksize);
+    int PIOc_File_is_Open(int ncid);
+
+    /* Set the IO node data buffer size limit. */
+    PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit);
+
+    /* Set the error hanlding for a file. */
+    int PIOc_Set_File_Error_Handling(int ncid, int method);
+
+    int PIOc_set_hint(int iosysid, const char *hint, const char *hintval);
+    int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
+                             float preemption);
+    int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                             float *preemptionp);
+
+    /* Dimensions. */
+    int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp);
+    int PIOc_inq_dimid(int ncid, const char *name, int *idp);
+    int PIOc_inq_dimname(int ncid, int dimid, char *name);
+    int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp);
+    int PIOc_rename_dim(int ncid, int dimid, const char *name);
+    int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp);
+
+    /* Variables. */
+    int PIOc_inq_varid(int ncid, const char *name, int *varidp);
+    int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp,
+                     int *nattsp);
+    int PIOc_inq_varname(int ncid, int varid, char *name);
+    int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep);
+    int PIOc_inq_varndims(int ncid, int varid, int *ndimsp);
+    int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp);
+    int PIOc_inq_varnatts(int ncid, int varid, int *nattsp);
+    int PIOc_def_var(int ncid, const char *name, nc_type xtype,  int ndims,
+                     const int *dimidsp, int *varidp);
+    int PIOc_set_fill(int ncid, int fillmode, int *old_modep);
+    int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
+    int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
+    int PIOc_rename_var(int ncid, int varid, const char *name);
+
+    /* These variable settings only apply to netCDF-4 files. */
+    int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
+                             int deflate_level);
+    int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
+                             int *deflate_levelp);
+    int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp);
+    int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp);
+    int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp);
+    int PIOc_def_var_endian(int ncid, int varid, int endian);
+    int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
+    int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
+                                 float preemption);
+    int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                                 float *preemptionp);
+
+    /* Attributes - misc. */
+    int PIOc_rename_att(int ncid, int varid, const char *name, const char *newname);
+    int PIOc_del_att(int ncid, int varid, const char *name);
+
+    /* Attributes - inquiry functions. */
+    int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
+                     PIO_Offset *lenp);
+    int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp);
+    int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp);
+    int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep);
+    int PIOc_inq_attname(int ncid, int varid, int attnum, char *name);
+
+    /* Attributes - writing. */
+    int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op);
+    int PIOc_put_att_text(int ncid, int varid, const char *name, PIO_Offset len, const char *op);
+    int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const signed char *op);
+    int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const short *op);
+    int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                         const int *op);
+    int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                          const long *op);
+    int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const float *op);
+    int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                            const double *op);
+    int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const unsigned char *op);
+    int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                            const unsigned short *op);
+    int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                          const unsigned int *op);
+    int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                              const long long *op);
+    int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
+                               PIO_Offset len, const unsigned long long *op);
+
+    /* Attributes - reading. */
+    int PIOc_get_att(int ncid, int varid, const char *name, void *ip);
+    int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip);
+    int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip);
+    int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip);
+    int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip);
+    int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip);
+    int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip);
+    int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip);
+    int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip);
+    int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip);
+    int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip);
+    int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip);
+    int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip);
+
+    /* Data reads - var. */
+    int PIOc_get_var(int ncid, int varid, void *buf);
+    int PIOc_get_var_text(int ncid, int varid, char *buf);
+    int PIOc_get_var_schar(int ncid, int varid, signed char *buf);
+    int PIOc_get_var_short(int ncid, int varid, short *buf);
+    int PIOc_get_var_int(int ncid, int varid, int *buf);
+    int PIOc_get_var_long(int ncid, int varid, long *buf);
+    int PIOc_get_var_float(int ncid, int varid, float *buf);
+    int PIOc_get_var_double(int ncid, int varid, double *buf);
+    int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf);
+    int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf);
+    int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf);
+    int PIOc_get_var_longlong(int ncid, int varid, long long *buf);
+    int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf);
+
+    /* Data writes - var. */
+    int PIOc_put_var(int ncid, int varid, const void *buf);
+    int PIOc_put_var_text(int ncid, int varid, const char *op);
+    int PIOc_put_var_schar(int ncid, int varid, const signed char *op);
+    int PIOc_put_var_short(int ncid, int varid, const short *op);
+    int PIOc_put_var_int(int ncid, int varid, const int *op);
+    int PIOc_put_var_long(int ncid, int varid, const long *op);
+    int PIOc_put_var_float(int ncid, int varid, const float *op);
+    int PIOc_put_var_double(int ncid, int varid, const double *op);
+    int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op);
+    int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op);
+    int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op);
+    int PIOc_put_var_longlong(int ncid, int varid, const long long *op);
+    int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op);
+
+    /* Data reads - var1. */
+    int PIOc_get_var1(int ncid, int varid, const PIO_Offset *index, void *buf);
+    int PIOc_get_var1_text(int ncid, int varid, const PIO_Offset *index, char *buf);
+    int PIOc_get_var1_schar(int ncid, int varid, const PIO_Offset *index, signed char *buf);
+    int PIOc_get_var1_short(int ncid, int varid, const PIO_Offset *index, short *buf);
+    int PIOc_get_var1_int(int ncid, int varid, const PIO_Offset *index, int *buf);
+    int PIOc_get_var1_long(int ncid, int varid, const PIO_Offset *index, long *buf);
+    int PIOc_get_var1_float(int ncid, int varid, const PIO_Offset *index, float *buf);
+    int PIOc_get_var1_double(int ncid, int varid, const PIO_Offset *index, double *buf);
+    int PIOc_get_var1_uchar(int ncid, int varid, const PIO_Offset *index, unsigned char *buf);
+    int PIOc_get_var1_ushort(int ncid, int varid, const PIO_Offset *index, unsigned short *buf);
+    int PIOc_get_var1_uint(int ncid, int varid, const PIO_Offset *index, unsigned int *buf);
+    int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index, long long *buf);
+    int PIOc_get_var1_ulonglong(int ncid, int varid, const PIO_Offset *index, unsigned long long *buf);
+
+    /* Data writes - var1. */
+    int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *buf);
+    int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op);
+    int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index, const signed char *op);
+    int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index, const short *op);
+    int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op);
+    int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *ip);
+    int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op);
+    int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index, const double *op);
+    int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
+                            const unsigned char *op);
+    int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
+                             const unsigned short *op);
+    int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
+                           const unsigned int *op);
+    int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index, const long long *op);
+    int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
+                                const unsigned long long *op);
+
+    /* Data reads - vara. */
+    int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf);
+    int PIOc_get_vara_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           char *buf);
+    int PIOc_get_vara_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            signed char *buf);
+    int PIOc_get_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            short *buf);
+    int PIOc_get_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          int *buf);
+    int PIOc_get_vara_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            float *buf);
+    int PIOc_get_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           long *buf);
+    int PIOc_get_vara_double(int ncid, int varid, const PIO_Offset *start,
+                             const PIO_Offset *count, double *buf);
+    int PIOc_get_vara_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            unsigned char *buf);
+    int PIOc_get_vara_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             unsigned short *buf);
+    int PIOc_get_vara_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           unsigned int *buf);
+    int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               long long *buf);
+    int PIOc_get_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                                unsigned long long *buf);
+
+    /* Data writes - vara. */
+    int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const void *buf);
+    int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const char *op);
+    int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const signed char *op);
+    int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const short *op);
+    int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const int *op);
+    int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const long *op);
+    int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const float *op);
+    int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const double *op);
+    int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const unsigned char *op);
+    int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                             const PIO_Offset *count, const unsigned short *op);
+    int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const unsigned int *op);
+    int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                               const PIO_Offset *count, const long long *op);
+    int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const unsigned long long *op);
+
+    /* Data reads - vars. */
+    int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, void *buf);
+    int PIOc_get_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, char *buf);
+    int PIOc_get_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, signed char *buf);
+    int PIOc_get_vars_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, short *buf);
+    int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const PIO_Offset *stride, int *buf);
+    int PIOc_get_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, long *buf);
+    int PIOc_get_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, float *buf);
+    int PIOc_get_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, double *buf);
+    int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, unsigned char *buf);
+    int PIOc_get_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, unsigned short *buf);
+    int PIOc_get_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, unsigned int *buf);
+    int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, long long *buf);
+    int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const PIO_Offset *stride,
+                                unsigned long long *buf);
+
+    /* Data writes - vars. */
+    int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const void *buf);
+    int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride, const char *op);
+    int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride,
+                            const signed char *op);
+    int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride, const short *op);
+    int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const PIO_Offset *stride, const int *op);
+    int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const float *op);
+    int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start,
+                             const PIO_Offset *count, const PIO_Offset *stride, const double *op);
+    int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const long *op);
+    int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const unsigned char *op);
+    int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const unsigned short *op);
+    int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const unsigned int *op);
+    int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, const long long *op);
+    int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const PIO_Offset *stride,
+                                const unsigned long long *op);
+
+    /* Varm functions are deprecated and should be used with extreme
+     * caution or not at all. Varm functions are not supported in
+     * async mode. */
+    int PIOc_put_varm(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const PIO_Offset *imap, const void *buf,
+                      PIO_Offset bufcount, MPI_Datatype buftype);
+    int PIOc_get_varm_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, signed char *buf);
+    int PIOc_put_varm_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap,
+                            const unsigned char *op);
+    int PIOc_put_varm_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, const short *op);
+    int PIOc_get_varm_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, short *buf);
+    int PIOc_get_varm_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                                const PIO_Offset *stride, const PIO_Offset *imap, unsigned long long *buf);
+    int PIOc_get_varm_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, unsigned short *buf);
+    int PIOc_get_varm_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, const PIO_Offset *imap, long long *buf);
+    int PIOc_put_varm_text(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride,
+                           const PIO_Offset *imap, const char *op);
+    int PIOc_put_varm_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, const unsigned short *op);
+    int PIOc_put_varm_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const PIO_Offset *stride,
+                                const PIO_Offset *imap, const unsigned long long *op);
+    int PIOc_put_varm_int(int ncid, int varid, const PIO_Offset *start,
+                          const PIO_Offset *count, const PIO_Offset *stride,
+                          const PIO_Offset *imap, const int *op);
+    int PIOc_put_varm_float(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride,
+                            const PIO_Offset *imap, const float *op);
+    int PIOc_put_varm_long(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride,
+                           const PIO_Offset *imap, const long *op);
+    int PIOc_put_varm_uint(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride,
+                           const PIO_Offset *imap, const unsigned int *op);
+    int PIOc_put_varm_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, const double *op);
+    int PIOc_put_varm_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, const signed char *op);
+    int PIOc_put_varm_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, const PIO_Offset *imap, const long long *op);
+    int PIOc_get_varm_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, double *buf);
+    int PIOc_get_varm_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const PIO_Offset *imap, char *buf);
+    int PIOc_get_varm_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const PIO_Offset *stride, const PIO_Offset *imap, int *buf);
+    int PIOc_get_varm_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const PIO_Offset *imap, unsigned int *buf);
+    int PIOc_get_varm(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const PIO_Offset *imap, void *buf,
+                      PIO_Offset bufcount, MPI_Datatype buftype);
+    int PIOc_get_varm_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, float *buf);
+    int PIOc_get_varm_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const PIO_Offset *imap, long *buf);
+#if defined(__cplusplus)
+}
+#endif
+
+#endif  // _PIO_H_
diff --git a/externals/pio2/src/clib/pio_c_get_template.c b/src/externals/pio2/src/clib/pio_c_get_template.c
similarity index 100%
rename from externals/pio2/src/clib/pio_c_get_template.c
rename to src/externals/pio2/src/clib/pio_c_get_template.c
diff --git a/externals/pio2/src/clib/pio_c_put_template.c b/src/externals/pio2/src/clib/pio_c_put_template.c
similarity index 100%
rename from externals/pio2/src/clib/pio_c_put_template.c
rename to src/externals/pio2/src/clib/pio_c_put_template.c
diff --git a/externals/pio2/src/clib/pio_c_template.c b/src/externals/pio2/src/clib/pio_c_template.c
similarity index 100%
rename from externals/pio2/src/clib/pio_c_template.c
rename to src/externals/pio2/src/clib/pio_c_template.c
diff --git a/src/externals/pio2/src/clib/pio_darray.c b/src/externals/pio2/src/clib/pio_darray.c
new file mode 100644
index 00000000000..4250d87ea38
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_darray.c
@@ -0,0 +1,626 @@
+/** @file
+ *
+ * Public functions that read and write distributed arrays in PIO.
+ *
+ * When arrays are distributed, each processor holds some of the
+ * array. Only by combining the distributed arrays from all processor
+ * can the full array be obtained.
+ *
+ * @author Jim Edwards
+ */
+#include 
+#include 
+#include 
+
+/* 10MB default limit. */
+PIO_Offset pio_buffer_size_limit = 10485760;
+
+/* Global buffer pool pointer. */
+void *CN_bpool = NULL;
+
+/* Maximum buffer usage. */
+PIO_Offset maxusage = 0;
+
+/**
+ * Set the PIO IO node data buffer size limit.
+ *
+ * The pio_buffer_size_limit will only apply to files opened after
+ * the setting is changed.
+ *
+ * @param limit the size of the buffer on the IO nodes
+ * @return The previous limit setting.
+ */
+PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
+{
+    PIO_Offset oldsize = pio_buffer_size_limit;
+
+    /* If the user passed a valid size, use it. */
+    if (limit > 0)
+        pio_buffer_size_limit = limit;
+
+    return oldsize;
+}
+
+/**
+ * Write one or more arrays with the same IO decomposition to the
+ * file.
+ *
+ * This funciton is similar to PIOc_write_darray(), but allows the
+ * caller to use their own data buffering (instead of using the
+ * buffering implemented in PIOc_write_darray()).
+ *
+ * @param ncid identifies the netCDF file.
+ * @param varids an array of length nvars containing the variable ids to
+ * be written.
+ * @param ioid the I/O description ID as passed back by
+ * PIOc_InitDecomp().
+ * @param nvars the number of variables to be written with this
+ * call.
+ * @param arraylen the length of the array to be written. This is the
+ * length of the distrubited array. That is, the length of the portion
+ * of the data that is on the processor. The same arraylen is used for
+ * all variables in the call.
+ * @param array pointer to the data to be written. This is a pointer
+ * to an array of arrays with the distributed portion of the array
+ * that is on this processor. There are nvars arrays of data, and each
+ * array of data contains one record worth of data for that variable.
+ * @param frame an array of length nvars with the frame or record
+ * dimension for each of the nvars variables in IOBUF
+ * @param fillvalue pointer to the fill value to be used for missing
+ * data. Ignored if NULL. If provided, must be the correct fill value
+ * for the variable. The correct fill value will be used if NULL is
+ * passed.
+ * @param flushtodisk non-zero to cause buffers to be flushed to disk.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen,
+                            void *array, const int *frame, void **fillvalue, bool flushtodisk)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    io_desc_t *iodesc;     /* Pointer to IO description information. */
+    int vsize;             /* size in bytes of the given data. */
+    int rlen;              /* total data buffer size. */
+    var_desc_t *vdesc0;    /* pointer to var_desc structure for each var. */
+    int mpierr;            /* Return code from MPI functions. */
+    int ierr;              /* Return code. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Check inputs. */
+    if (nvars <= 0 || !varids)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+    for (int v = 0; v < nvars; v++)
+        if (varids[v] < 0 || varids[v] > PIO_MAX_VARS)
+            return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld flushtodisk = %d",
+         ncid, ioid, nvars, arraylen, flushtodisk));
+
+    /* Check that we can write to this file. */
+    if (! (file->mode & PIO_WRITE))
+        return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__);
+
+    /* Get iodesc. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* For netcdf serial writes we collect the data on io nodes and
+     * then move that data one node at a time to the io master node
+     * and write (or read). The buffer size on io task 0 must be as
+     * large as the largest used to accommodate this serial io
+     * method. */
+    vdesc0 = file->varlist + varids[0];
+    pioassert(!vdesc0->iobuf, "Attempt to overwrite existing io buffer",__FILE__, __LINE__);
+
+    /* ??? */
+    /*   rlen = iodesc->llen*nvars; */
+    rlen = 0;
+    if (iodesc->llen > 0)
+        rlen = iodesc->maxiobuflen * nvars;
+
+    /* Currently there are two rearrangers box=1 and subset=2. There
+     * is never a case where rearranger==0. */
+    LOG((2, "iodesc->rearranger = %d iodesc->needsfill = %d\n", iodesc->rearranger,
+         iodesc->needsfill));
+    if (iodesc->rearranger > 0)
+    {
+        if (rlen > 0)
+        {
+            if ((mpierr = MPI_Type_size(iodesc->basetype, &vsize)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+            LOG((3, "vsize = %d", vsize));
+
+            /* Allocate memory for the variable buffer. */
+            if (!(vdesc0->iobuf = bget((size_t)vsize * (size_t)rlen)))
+                piomemerror(ios, (size_t)rlen * (size_t)vsize, __FILE__, __LINE__);
+            LOG((3, "allocated %ld bytes for variable buffer", (size_t)rlen * (size_t)vsize));
+
+            /* If data are missing for the BOX rearranger, insert fill values. */
+            if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX)
+                for (int nv = 0; nv < nvars; nv++)
+                    for (int i = 0; i < iodesc->maxiobuflen; i++)
+                        memcpy(&((char *)vdesc0->iobuf)[vsize * (i + nv * iodesc->maxiobuflen)],
+                               &((char *)fillvalue)[nv * vsize], vsize);
+        }
+
+        /* Move data from compute to IO tasks. */
+        if ((ierr = rearrange_comp2io(ios, iodesc, array, vdesc0->iobuf, nvars)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+    }
+    
+    /* Write the darray based on the iotype. */
+    LOG((2, "about to write darray for iotype = %d", file->iotype));
+    switch (file->iotype)
+    {
+    case PIO_IOTYPE_NETCDF4P:
+    case PIO_IOTYPE_PNETCDF:
+        if ((ierr = pio_write_darray_multi_nc(file, nvars, varids, iodesc->ndims, iodesc->basetype,
+                                              iodesc->maxregions, iodesc->firstregion, iodesc->llen,
+                                              iodesc->num_aiotasks, vdesc0->iobuf, frame)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        break;
+    case PIO_IOTYPE_NETCDF4C:
+    case PIO_IOTYPE_NETCDF:
+        if ((ierr = pio_write_darray_multi_nc_serial(file, nvars, varids, iodesc->ndims, iodesc->basetype,
+                                                     iodesc->maxregions, iodesc->firstregion, iodesc->llen,
+                                                     iodesc->num_aiotasks, vdesc0->iobuf, frame)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        break;
+    default:
+        return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
+    }
+
+    /* For PNETCDF the iobuf is freed in flush_output_buffer() */
+    if (file->iotype != PIO_IOTYPE_PNETCDF)
+    {
+        /* Release resources. */
+        if (vdesc0->iobuf)
+        {
+            brel(vdesc0->iobuf);
+            vdesc0->iobuf = NULL;
+        }
+    }
+
+    /* The box rearranger will always have data (it could be fill
+     * data) to fill the entire array - that is the aggregate start
+     * and count values will completely describe one unlimited
+     * dimension unit of the array. For the subset method this is not
+     * necessarily the case, areas of missing data may never be
+     * written. In order to make sure that these areas are given the
+     * missing value a 'holegrid' is used to describe the missing
+     * points. This is generally faster than the netcdf method of
+     * filling the entire array with missing values before overwriting
+     * those values later. */
+    if (iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill)
+    {
+        LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars,
+             iodesc->holegridsize, iodesc->needsfill));
+
+        if (vdesc0->fillbuf)
+            piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__);
+
+        /* Get a buffer. */
+	if (ios->io_rank == 0)
+	    vdesc0->fillbuf = bget(iodesc->maxholegridsize * vsize * nvars);
+	else if (iodesc->holegridsize > 0)
+	    vdesc0->fillbuf = bget(iodesc->holegridsize * vsize * nvars);
+
+        /* copying the fill value into the data buffer for the box
+         * rearranger. This will be overwritten with data where
+         * provided. */
+        for (int nv = 0; nv < nvars; nv++)
+            for (int i = 0; i < iodesc->holegridsize; i++)
+                memcpy(&((char *)vdesc0->fillbuf)[vsize * (i + nv * iodesc->holegridsize)],
+                       &((char *)fillvalue)[vsize * nv], vsize);
+
+        /* Write the darray based on the iotype. */
+        switch (file->iotype)
+        {
+        case PIO_IOTYPE_PNETCDF:
+        case PIO_IOTYPE_NETCDF4P:
+            if ((ierr = pio_write_darray_multi_nc(file, nvars, varids,
+                                                  iodesc->ndims, iodesc->basetype, iodesc->maxfillregions,
+                                                  iodesc->fillregion, iodesc->holegridsize,
+                                                  iodesc->num_aiotasks, vdesc0->fillbuf, frame)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+        case PIO_IOTYPE_NETCDF:
+            if ((ierr = pio_write_darray_multi_nc_serial(file, nvars, varids, iodesc->ndims, iodesc->basetype,
+                                                         iodesc->maxfillregions, iodesc->fillregion,
+                                                         iodesc->holegridsize,
+                                                         iodesc->num_aiotasks, vdesc0->fillbuf, frame)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            break;
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+
+        /* For PNETCDF fillbuf is freed in flush_output_buffer() */
+        if (file->iotype != PIO_IOTYPE_PNETCDF)
+        {
+            /* Free resources. */
+            if (vdesc0->fillbuf)
+            {
+                brel(vdesc0->fillbuf);
+                vdesc0->fillbuf = NULL;
+            }
+        }
+    }
+
+    /* Flush data to disk. */
+    if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
+        if ((ierr = flush_output_buffer(file, flushtodisk, 0)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write a distributed array to the output file.
+ *
+ * This routine aggregates output on the compute nodes and only sends
+ * it to the IO nodes when the compute buffer is full or when a flush
+ * is triggered.
+ *
+ * @param ncid the ncid of the open netCDF file.
+ * @param varid the ID of the variable that these data will be written
+ * to.
+ * @param ioid the I/O description ID as passed back by
+ * PIOc_InitDecomp().
+ * @param arraylen the length of the array to be written.  This should
+ * be at least the length of the local component of the distrubited
+ * array. (Any values beyond length of the local component will be
+ * ignored.)
+ * @param array pointer to an array of length arraylen with the data
+ * to be written. This is a pointer to the distributed portion of the
+ * array that is on this task.
+ * @param fillvalue pointer to the fill value to be used for
+ * missing data.
+ * @returns 0 for success, non-zero error code for failure.
+ * @ingroup PIO_write_darray
+ */
+int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
+                      void *fillvalue)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Info about file we are writing to. */
+    io_desc_t *iodesc;     /* The IO description. */
+    var_desc_t *vdesc;     /* Info about the var being written. */
+    void *bufptr;          /* A data buffer. */
+    MPI_Datatype vtype;    /* The MPI type of the variable. */
+    wmulti_buffer *wmb;    /* The write multi buffer for one or more vars. */
+    int tsize;             /* Size of MPI type. */
+    bool recordvar;        /* True if this is a record variable. */
+    int needsflush = 0;    /* True if we need to flush buffer. */
+    bufsize totfree;       /* Amount of free space in the buffer. */
+    bufsize maxfree;       /* Max amount of free space in buffer. */
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI functions. */
+    int ierr = PIO_NOERR;  /* Return code. */
+
+    LOG((1, "PIOc_write_darray ncid = %d varid = %d ioid = %d arraylen = %d",
+         ncid, varid, ioid, arraylen));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Can we write to this file? */
+    if (!(file->mode & PIO_WRITE))
+        return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__);
+
+    /* Get decomposition information. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Get var description. */
+    vdesc = file->varlist + varid;
+    LOG((2, "vdesc record %d ndims %d nreqs %d", vdesc->record, vdesc->ndims, vdesc->nreqs));
+
+    /* Is this a record variable? */
+    recordvar = vdesc->record >= 0 ? true : false;
+    LOG((3, "recordvar = %d", recordvar));
+
+    /* Check that the local size of the variable passed in matches the
+     * size expected by the io descriptor. */
+    if (arraylen < iodesc->ndof)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    if (iodesc->ndof != arraylen)
+        LOG((1, "User supplied array is larger than expected, arraylen != iodesc->ndof"));
+
+    /* Get a pointer to the buffer space for this file. It will hold
+     * data from one or more variables that fit the same
+     * description. */
+    wmb = &file->buffer;
+
+    /* If the ioid is not initialized, set it. For non record vars,
+     * use the negative ??? */
+    if (wmb->ioid == -1)
+        wmb->ioid = recordvar ? ioid : -ioid;
+    else
+    {
+        /* Handle record and non-record variables differently. */
+        if (recordvar)
+        {
+            /* Moving to the end of the wmb linked list to add the
+             * current variable. ??? */
+            while(wmb->next && wmb->ioid != ioid)
+                if (wmb->next)
+                    wmb = wmb->next;
+#ifdef _PNETCDF
+            /* Do we still need the commented code below? ??? */
+            /* flush the previous record before starting a new one. this is collective */
+            /*       if (vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) ||
+                     (wmb->frame != NULL && vdesc->record != wmb->frame[0])){
+                     needsflush = 2;  // flush to disk
+                     } */
+#endif
+        }
+        else
+        {
+            /* Move to end of list. */
+            while(wmb->next && wmb->ioid != -(ioid))
+                if (wmb->next)
+                    wmb = wmb->next;
+        }
+    }
+
+    /* The write multi buffer wmulti_buffer is the cache on compute
+       nodes that will collect and store multiple variables before
+       sending them to the io nodes. Aggregating variables in this way
+       leads to a considerable savings in communication
+       expense. Variables in the wmb array must have the same
+       decomposition and base data size and we also need to keep track
+       of whether each is a recordvar (has an unlimited dimension) or
+       not. */
+    if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid)))
+    {
+        /* Allocate a buffer. */
+        if (!(wmb->next = bget((bufsize)sizeof(wmulti_buffer))))
+            piomemerror(ios, sizeof(wmulti_buffer), __FILE__, __LINE__);
+        LOG((3, "allocated multi-buffer"));
+
+        /* Set pointer to newly allocated buffer and initialize.*/
+        wmb = wmb->next;
+        wmb->next = NULL;
+        wmb->ioid = recordvar ? ioid : -ioid;
+        wmb->validvars = 0;
+        wmb->arraylen = arraylen;
+        wmb->vid = NULL;
+        wmb->data = NULL;
+        wmb->frame = NULL;
+        wmb->fillvalue = NULL;
+    }
+
+    /* Get the size of the MPI type. */
+    if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "wmb->validvars = %d arraylen = %d tsize = %d\n", wmb->validvars,
+         arraylen, tsize));
+
+    /* At this point wmb should be pointing to a new or existing buffer
+       so we can add the data. */
+
+    /* Find out how much free, contiguous space is available. */
+    bfreespace(&totfree, &maxfree);
+
+    /* maxfree is the available memory. If that is < 10% greater than
+     * the size of the current request needsflush is true. */
+    if (needsflush == 0)
+        needsflush = (maxfree <= 1.1 * (1 + wmb->validvars) * arraylen * tsize);
+
+    /* Tell all tasks on the computation communicator whether we need
+     * to flush data. */
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1,  MPI_INT,  MPI_MAX, ios->comp_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "needsflush = %d", needsflush));
+
+    /* Flush data if needed. */
+    if (needsflush > 0)
+    {
+        LOG((2, "maxfree = %ld wmb->validvars = %d (1 + wmb->validvars) * arraylen * tsize = %ld totfree = %ld\n",
+             maxfree, wmb->validvars, (1 + wmb->validvars) * arraylen * tsize, totfree));
+
+#ifdef PIO_ENABLE_LOGGING
+        /* Collect a debug report about buffer. */
+        cn_buffer_report(ios, true);
+#endif /* PIO_ENABLE_LOGGING */
+
+        /* If needsflush == 2 flush to disk otherwise just flush to io node. */
+        if ((ierr = flush_buffer(ncid, wmb, needsflush == 2)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+    }
+
+    /* Get memory for data. */
+    if (arraylen > 0)
+    {
+        if (!(wmb->data = bgetr(wmb->data, (1 + wmb->validvars) * arraylen * tsize)))
+            piomemerror(ios, (1 + wmb->validvars) * arraylen * tsize, __FILE__, __LINE__);
+        LOG((2, "got %ld bytes for data", (1 + wmb->validvars) * arraylen * tsize));
+    }
+
+    /* vid is an array of variable ids in the wmb list, grow the list
+     * and add the new entry. */
+    if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->validvars))))
+        piomemerror(ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__);
+
+    /* wmb->frame is the record number, we assume that the variables
+     * in the wmb list may not all have the same unlimited dimension
+     * value although they usually do. */
+    if (vdesc->record >= 0)
+        if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->validvars))))
+            piomemerror(ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__);
+
+    /* If we need a fill value, get it. */
+    if (iodesc->needsfill)
+    {
+        /* Get memory to hold fill value. */
+        if (!(wmb->fillvalue = bgetr(wmb->fillvalue, tsize * (1 + wmb->validvars))))
+            piomemerror(ios, (1 + wmb->validvars) * tsize, __FILE__, __LINE__);
+
+        /* If the user passed a fill value, use that, otherwise use
+         * the default fill value of the netCDF type. Copy the fill
+         * value to the buffer. */
+        if (fillvalue)
+        {
+            memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, fillvalue, tsize);
+            LOG((3, "copied user-provided fill value tsize = %d", tsize));
+        }
+        else
+        {
+            vtype = (MPI_Datatype)iodesc->basetype;
+            LOG((3, "caller did not provide fill value vtype = %d", vtype));
+            if (vtype == MPI_INT)
+            {
+                int fill = PIO_FILL_INT;
+                memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize);
+            }
+            else if (vtype == MPI_FLOAT)
+            {
+                float fill = PIO_FILL_FLOAT;
+                memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize);
+            }
+            else if (vtype == MPI_DOUBLE)
+            {
+                double fill = PIO_FILL_DOUBLE;
+                memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize);
+            }
+            else if (vtype == MPI_CHARACTER)
+            {
+                char fill = PIO_FILL_CHAR;
+                memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize);
+            }
+            else
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    /* Tell the buffer about the data it is getting. */
+    wmb->arraylen = arraylen;
+    wmb->vid[wmb->validvars] = varid;
+
+    /* Copy the user-provided data to the buffer. */
+    bufptr = (void *)((char *)wmb->data + arraylen * tsize * wmb->validvars);
+    if (arraylen > 0)
+    {
+        memcpy(bufptr, array, arraylen * tsize);
+        LOG((3, "copied %ld bytes of user data", arraylen * tsize));
+    }
+
+    /* Add the unlimited dimension value of this variable to the frame
+     * array in wmb. */
+    if (wmb->frame)
+        wmb->frame[wmb->validvars] = vdesc->record;
+    wmb->validvars++;
+
+    LOG((2, "wmb->validvars = %d iodesc->maxbytes / tsize = %d iodesc->ndof = %d iodesc->llen = %d",
+         wmb->validvars, iodesc->maxbytes / tsize, iodesc->ndof, iodesc->llen));
+
+    /* Call the sync when ??? */
+    if (wmb->validvars >= iodesc->maxbytes / tsize)
+        PIOc_sync(ncid);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read a field from a file to the IO library.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID to be read
+ * @param ioid: the I/O description ID as passed back by
+ * PIOc_InitDecomp().
+ * @param arraylen: the length of the array to be read. This
+ * is the length of the distrubited array. That is, the length of
+ * the portion of the data that is on the processor.
+ * @param array: pointer to the data to be read. This is a
+ * pointer to the distributed portion of the array that is on this
+ * processor.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_read_darray
+ */
+int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
+                     void *array)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    io_desc_t *iodesc;     /* Pointer to IO description information. */
+    void *iobuf = NULL;    /* holds the data as read on the io node. */
+    size_t rlen = 0;       /* the length of data in iobuf. */
+    int tsize;          /* Total size. */
+    int mpierr;         /* Return code from MPI functions. */
+    int ierr;           /* Return code. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Get the iodesc. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* ??? */
+    if (ios->iomaster == MPI_ROOT)
+        rlen = iodesc->maxiobuflen;
+    else
+        rlen = iodesc->llen;
+
+    /* Is a rearranger in use? */
+    if (iodesc->rearranger > 0)
+    {
+        if (ios->ioproc && rlen > 0)
+        {
+            /* Get the MPI type size. */
+            if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+            /* Allocate a buffer for one record. */
+            if (!(iobuf = bget((size_t)tsize * rlen)))
+                piomemerror(ios, rlen * (size_t)tsize, __FILE__, __LINE__);
+        }
+    }
+    else
+    {
+        iobuf = array;
+    }
+
+    /* Call the correct darray read function based on iotype. */
+    switch (file->iotype)
+    {
+    case PIO_IOTYPE_NETCDF:
+    case PIO_IOTYPE_NETCDF4C:
+        if ((ierr = pio_read_darray_nc_serial(file, iodesc, varid, iobuf)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        break;
+    case PIO_IOTYPE_PNETCDF:
+    case PIO_IOTYPE_NETCDF4P:
+        if ((ierr = pio_read_darray_nc(file, iodesc, varid, iobuf)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        break;
+    default:
+        return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
+    }
+
+    /* If a rearranger was specified, rearrange the data. */
+    if (iodesc->rearranger > 0)
+    {
+        if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        /* Free the buffer. */
+        if (rlen > 0)
+            brel(iobuf);
+    }
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_darray_int.c b/src/externals/pio2/src/clib/pio_darray_int.c
new file mode 100644
index 00000000000..ac19b37e198
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_darray_int.c
@@ -0,0 +1,1407 @@
+/** @file
+ *
+ * Private functions to help read and write distributed arrays in PIO.
+ *
+ * When arrays are distributed, each processor holds some of the
+ * array. Only by combining the distributed arrays from all processor
+ * can the full array be obtained.
+ *
+ * @author Jim Edwards
+ */
+
+#include 
+#include 
+#include 
+
+/* 10MB default limit. */
+extern PIO_Offset pio_buffer_size_limit;
+
+/* Initial size of compute buffer. */
+bufsize pio_cnbuffer_limit = 33554432;
+
+/* Global buffer pool pointer. */
+extern void *CN_bpool;
+
+/* Maximum buffer usage. */
+extern PIO_Offset maxusage;
+
+/**
+ * Initialize the compute buffer to size pio_cnbuffer_limit.
+ *
+ * This routine initializes the compute buffer pool if the bget memory
+ * management is used. If malloc is used (that is, PIO_USE_MALLOC is
+ * non zero), this function does nothing.
+ *
+ * @param ios pointer to the iosystem descriptor which will use the
+ * new buffer.
+ * @returns 0 for success, error code otherwise.
+ */
+int compute_buffer_init(iosystem_desc_t *ios)
+{
+#if !PIO_USE_MALLOC
+
+    if (!CN_bpool)
+    {
+        if (!(CN_bpool = malloc(pio_cnbuffer_limit)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        bpool(CN_bpool, pio_cnbuffer_limit);
+        if (!CN_bpool)
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        bectl(NULL, malloc, free, pio_cnbuffer_limit);
+    }
+#endif
+    LOG((2, "compute_buffer_init CN_bpool = %d", CN_bpool));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write a set of one or more aggregated arrays to output file. This
+ * function is only used with parallel-netcdf and netcdf-4 parallel
+ * iotypes. Serial io types use pio_write_darray_multi_nc_serial().
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param nvars the number of variables to be written with this
+ * decomposition.
+ * @param vid: an array of the variable ids to be written.
+ * @param iodesc_ndims: the number of dimensions explicitly in the
+ * iodesc.
+ * @param basetype the basic type of the minimal data unit
+ * @param maxregions max number of blocks to be written from
+ * this iotask.
+ * @param firstregion pointer to the first element of a linked
+ * list of region descriptions. May be NULL.
+ * @param llen length of the iobuffer on this task for a single
+ * field.
+ * @param num_aiotasks actual number of iotasks participating.
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @param frame the frame or record dimension for each of the nvars
+ * variables in iobuf.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims,
+                              MPI_Datatype basetype, int maxregions, io_region *firstregion,
+                              PIO_Offset llen, int num_aiotasks, void *iobuf,
+                              const int *frame)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Pointer to var info struct. */
+    int fndims;            /* Number of dims for this var in the file. */
+    int dsize;             /* Data size (for one region). */
+    int tsize;             /* Size of MPI type. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && vid && vid[0] >= 0 && vid[0] <= PIO_MAX_VARS,
+              "invalid input", __FILE__, __LINE__);
+
+    LOG((1, "pio_write_darray_multi_nc nvars = %d iodesc_ndims = %d basetype = %d "
+         "maxregions = %d llen = %d num_aiotasks = %d", nvars, iodesc_ndims,
+         basetype, maxregions, llen, num_aiotasks));
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:write_darray_multi_nc");
+#endif
+
+    /* Get pointer to iosystem. */
+    ios = file->iosystem;
+
+    /* Point to var description scruct for first var. */
+    vdesc = file->varlist + vid[0];
+
+    /* If async is in use, send message to IO master task. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = 0;
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->pio_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Find out how many dims this variable has. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid[0], &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Find out the size of the MPI type. */
+    if ((mpierr = MPI_Type_size(basetype, &tsize)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "fndims = %d tsize = %d", fndims, tsize));
+
+    /* If this is an IO task write the data. */
+    if (ios->ioproc)
+    {
+        io_region *region = firstregion;
+        int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
+        void *bufptr;
+        size_t start[fndims];
+        size_t count[fndims];
+        int ndims = iodesc_ndims;
+        PIO_Offset *startlist[maxregions]; /* Array of start arrays for ncmpi_iput_varn(). */
+        PIO_Offset *countlist[maxregions]; /* Array of count  arrays for ncmpi_iput_varn(). */
+
+        LOG((3, "maxregions = %d", maxregions));
+
+        /* Process each region of data to be written. */
+        for (int regioncnt = 0; regioncnt < maxregions; regioncnt++)
+        {
+            /* Init start/count arrays to zero. */
+            for (int i = 0; i < fndims; i++)
+            {
+                start[i] = 0;
+                count[i] = 0;
+            }
+
+            if (region)
+            {
+                if (vdesc->record >= 0)
+                {
+                    /* This is a record based multidimensional
+                     * array. Figure out start/count for all but the
+                     * record dimension (dimid 0). */
+                    for (int i = fndims - ndims; i < fndims; i++)
+                    {
+                        start[i] = region->start[i - (fndims - ndims)];
+                        count[i] = region->count[i - (fndims - ndims)];
+                    }
+
+                    /* Now figure out start/count for record dimension. */
+                    if (fndims > 1 && ndims < fndims && count[1] > 0)
+                    {
+                        count[0] = 1;
+                        start[0] = frame[0];
+                    }
+                    else if (fndims == ndims)
+                    {
+                        /* ??? */
+                        start[0] += vdesc->record;
+                    }
+                }
+                else
+                {
+                    /* This is a non record variable. */
+                    for (int i = 0; i < ndims; i++)
+                    {
+                        start[i] = region->start[i];
+                        count[i] = region->count[i];
+                    }
+                }
+#if PIO_ENABLE_LOGGING
+                /* Log arrays for debug purposes. */
+                for (int i = 0; i < ndims; i++)
+                    LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i]));
+#endif /* PIO_ENABLE_LOGGING */
+            }
+
+            /* IO tasks will run the netCDF/pnetcdf functions to write the data. */
+            switch (file->iotype)
+            {
+#ifdef _NETCDF4
+            case PIO_IOTYPE_NETCDF4P:
+                /* For each variable to be written. */
+                for (int nv = 0; nv < nvars; nv++)
+                {
+                    /* Set the start of the record dimension. (Hasn't
+                     * this already been set above ???) */
+                    if (vdesc->record >= 0 && ndims < fndims)
+                        start[0] = frame[nv];
+
+                    /* If there is data for this region, get a pointer to it. */
+                    if (region)
+                        bufptr = (void *)((char *)iobuf + tsize * (nv * llen + region->loffset));
+
+                    /* Ensure collective access. */
+                    ierr = nc_var_par_access(file->fh, vid[nv], NC_COLLECTIVE);
+
+                    /* Write the data for this variable. */
+                    if (!ierr)
+                        ierr = nc_put_vara(file->fh, vid[nv], (size_t *)start, (size_t *)count, bufptr);
+                }
+                break;
+#endif
+#ifdef _PNETCDF
+            case PIO_IOTYPE_PNETCDF:
+                /* Get the total number of data elements we are
+                 * writing for this region. */
+                dsize = 1;
+                for (int i = 0; i < fndims; i++)
+                    dsize *= count[i];
+                LOG((3, "dsize = %d", dsize));
+
+                /* For pnetcdf's ncmpi_iput_varn() function, we need
+                 * to provide arrays of arrays for start/count. */
+                if (dsize > 0)
+                {
+                    /* Allocate storage for start/count arrays for
+                     * this region. */
+                    if (!(startlist[rrcnt] = calloc(fndims, sizeof(PIO_Offset))))
+                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+                    if (!(countlist[rrcnt] = calloc(fndims, sizeof(PIO_Offset))))
+                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+                    /* Copy the start/count arrays for this region. */
+                    for (int i = 0; i < fndims; i++)
+                    {
+                        startlist[rrcnt][i] = start[i];
+                        countlist[rrcnt][i] = count[i];
+                        LOG((3, "startlist[%d][%d] = %d countlist[%d][%d] = %d", rrcnt, i,
+                             startlist[rrcnt][i], rrcnt, i, countlist[rrcnt][i]));
+                    }
+                    rrcnt++;
+                }
+
+                /* Do this when we reach the last region. */
+                if (regioncnt == maxregions - 1)
+                {
+                    /* For each variable to be written. */
+                    for (int nv = 0; nv < nvars; nv++)
+                    {
+                        /* Get the var info. */
+                        vdesc = file->varlist + vid[nv];
+
+                        /* If this is a record var, set the start for
+                         * the record dimension. */
+                        if (vdesc->record >= 0 && ndims < fndims)
+                            for (int rc = 0; rc < rrcnt; rc++)
+                                startlist[rc][0] = frame[nv];
+
+                        /* Get a pointer to the data. */
+                        bufptr = (void *)((char *)iobuf + nv * tsize * llen);
+
+                        /* ??? */
+                        int reqn = 0;
+                        if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0 )
+                        {
+                            if (!(vdesc->request = realloc(vdesc->request, sizeof(int) *
+                                                           (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK))))
+                                return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+                            for (int i = vdesc->nreqs; i < vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK; i++)
+                                vdesc->request[i] = NC_REQ_NULL;
+                            reqn = vdesc->nreqs;
+                        }
+                        else
+                            while(vdesc->request[reqn] != NC_REQ_NULL)
+                                reqn++;
+
+                        /* Write, in non-blocking fashion, a list of subarrays. */
+                        LOG((3, "about to call ncmpi_iput_varn() vid[%d] = %d rrcnt = %d, llen = %d",
+                             nv, vid[nv], rrcnt, llen));
+                        ierr = ncmpi_iput_varn(file->fh, vid[nv], rrcnt, startlist, countlist,
+                                               bufptr, llen, basetype, vdesc->request + reqn);
+
+                        /* keeps wait calls in sync */
+                        if (vdesc->request[reqn] == NC_REQ_NULL)
+                            vdesc->request[reqn] = PIO_REQ_NULL;
+
+                        vdesc->nreqs += reqn + 1;
+
+                    }
+
+                    /* Free resources. */
+                    for (int i = 0; i < rrcnt; i++)
+                    {
+                        free(startlist[i]);
+                        free(countlist[i]);
+                    }
+                }
+                break;
+#endif
+            default:
+                return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+            }
+
+            /* Go to next region. */
+            if (region)
+                region = region->next;
+        } /* next regioncnt */
+    } /* endif (ios->ioproc) */
+
+    /* Check the return code from the netCDF/pnetcdf call. */
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:write_darray_multi_nc");
+#endif
+
+    return ierr;
+}
+
+/**
+ * Write a set of one or more aggregated arrays to output file in
+ * serial mode. This function is called for netCDF classic and
+ * netCDF-4 serial iotypes. Parallel iotypes use
+ * pio_write_darray_multi_nc().
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to.
+ * @param nvars the number of variables to be written with this
+ * decomposition.
+ * @param vid an array of the variable ids to be written
+ * @param iodesc_ndims the number of dimensions explicitly in the
+ * iodesc.
+ * @param basetype the basic type of the minimal data unit
+ * @param maxregions max number of blocks to be written from this
+ * iotask.
+ * @param firstregion pointer to the first element of a linked
+ * list of region descriptions. May be NULL.
+ * @param llen length of the iobuffer on this task for a single
+ * field.
+ * @param num_aiotasks actual number of iotasks participating
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @param frame the record dimension for each of the nvars variables
+ * in iobuf.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims,
+                                     MPI_Datatype basetype, int maxregions, io_region *firstregion,
+                                     PIO_Offset llen, int num_aiotasks, void *iobuf,
+                                     const int *frame)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Contains info about the variable. */
+    int dsize;             /* Size in bytes of one element of basetype. */
+    int fndims;            /* Number of dims in the var in the file. */
+    int tsize;             /* Size of the MPI type, in bytes. */
+    MPI_Status status;     /* Recv status for MPI. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;              /* Return code. */
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && vid && vid[0] >= 0 && vid[0] <= PIO_MAX_VARS,
+              "invalid input", __FILE__, __LINE__);
+
+    LOG((1, "pio_write_darray_multi_nc_serial nvars = %d iodesc_ndims = %d basetype = %d "
+         "maxregions = %d llen = %d num_aiotasks = %d", nvars, iodesc_ndims,
+         basetype, maxregions, llen, num_aiotasks));
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:write_darray_multi_nc_serial");
+#endif
+
+    /* Get the iosystem info. */
+    ios = file->iosystem;
+
+    /* Get the var info. */
+    vdesc = file->varlist + vid[0];
+
+    LOG((2, "vdesc record %d ndims %d nreqs %d ios->async_interface = %d", vdesc->record,
+         vdesc->ndims, vdesc->nreqs, ios->async_interface));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = 0;
+
+            if (ios->comp_rank == 0)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->pio_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Get the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid[0], &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Get the size of the type. */
+    if ((mpierr = MPI_Type_size(basetype, &tsize)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((3, "fndims = %d tsize = %d", fndims, tsize));
+
+    /* Only IO tasks participate in this code. */
+    if (ios->ioproc)
+    {
+        io_region *region;
+        void *bufptr;
+        size_t tmp_start[fndims * maxregions]; /* A start array for each region. */
+        size_t tmp_count[fndims * maxregions]; /* A count array for each region. */
+
+        LOG((3, "maxregions = %d", maxregions));
+
+        /* Fill the tmp_start and tmp_count arrays, which contain the
+         * start and count arrays for all regions. */
+        region = firstregion;
+        for (int regioncnt = 0; regioncnt < maxregions; regioncnt++)
+        {
+            /* Initialize the start/count arrays for this region to 0. */
+            for (int i = 0; i < fndims; i++)
+            {
+                tmp_start[i + regioncnt * fndims] = 0;
+                tmp_count[i + regioncnt * fndims] = 0;
+            }
+
+            if (region)
+            {
+                if (vdesc->record >= 0)
+                {
+                    /* This is a record based multidimensional
+                     * array. Copy start/count for non-record
+                     * dimensions. */
+                    for (int i = fndims - iodesc_ndims; i < fndims; i++)
+                    {
+                        tmp_start[i + regioncnt * fndims] = region->start[i - (fndims - iodesc_ndims)];
+                        tmp_count[i + regioncnt * fndims] = region->count[i - (fndims - iodesc_ndims)];
+                        LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims,
+                             tmp_start[i + regioncnt * fndims], i + regioncnt * fndims,
+                             tmp_count[i + regioncnt * fndims]));
+                    }
+                }
+                else
+                {
+                    /* This is not a record based multidimensional array. */
+                    for (int i = 0; i < iodesc_ndims; i++)
+                    {
+                        tmp_start[i + regioncnt * fndims] = region->start[i];
+                        tmp_count[i + regioncnt * fndims] = region->count[i];
+                        LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims,
+                             tmp_start[i + regioncnt * fndims], i + regioncnt * fndims,
+                             tmp_count[i + regioncnt * fndims]));
+                    }
+                }
+
+                /* Move to next region. */
+                region = region->next;
+
+            } /* endif region */
+        } /* next regioncnt */
+
+        /* Tasks other than 0 will send their data to task 0. */
+        if (ios->io_rank > 0)
+        {
+            /* Do a handshake. */
+            if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+            /* Send local length of iobuffer for each field (all
+             * fields are the same length). */
+            if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+            LOG((3, "sent llen = %d", llen));
+
+            /* Send the number of data regions, the start/count for
+             * all regions, and the data buffer with all the data. */
+            if (llen > 0)
+            {
+                if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0, ios->io_rank + 2 * ios->num_iotasks,
+                                       ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0, ios->io_rank + 3 * ios->num_iotasks,
+                                       ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(iobuf, nvars * llen, basetype, 0, ios->io_rank + 4 * ios->num_iotasks, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                LOG((3, "sent data for maxregions = %d", maxregions));
+            }
+        }
+        else
+        {
+            /* Task 0 will receive data from all other IO tasks. */
+            size_t rlen;    /* Length of IO buffer on this task. */
+            int rregions;   /* Number of regions in buffer for this task. */
+            size_t start[fndims], count[fndims];
+            size_t loffset;
+
+            /* Get the size of the MPI data type. */
+            if ((mpierr = MPI_Type_size(basetype, &dsize)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+            LOG((3, "dsize = %d", dsize));
+
+            /* For each of the other tasks that are using this task
+             * for IO. */
+            for (int rtask = 0; rtask < ios->num_iotasks; rtask++)
+            {
+                /* From the remote tasks, we send information about
+                 * the data regions. and also the data. */
+                if (rtask)
+                {
+                    /* handshake - tell the sending task I'm ready */
+                    if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm)))
+                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+                    /* Get length of iobuffer for each field on this
+                     * task (all fields are the same length). */
+                    if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status)))
+                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    LOG((3, "received rlen = %d", rlen));
+
+                    /* Get the number of regions, the start/count
+                     * values for all regions, and the data buffer. */
+                    if (rlen > 0)
+                    {
+                        if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks,
+                                               ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask, rtask + 2 * ios->num_iotasks,
+                                               ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask, rtask + 3 * ios->num_iotasks,
+                                               ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(iobuf, nvars * rlen, basetype, rtask, rtask + 4 * ios->num_iotasks, ios->io_comm,
+                                               &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        LOG((3, "received data rregions = %d fndims = %d", rregions, fndims));
+                    }
+                }
+                else /* task 0 */
+                {
+                    rlen = llen;
+                    rregions = maxregions;
+                }
+                LOG((3, "rtask = %d rlen = %d rregions = %d", rtask, rlen, rregions));
+
+                /* If there is data from this task, write it. */
+                if (rlen > 0)
+                {
+                    loffset = 0;
+                    for (int regioncnt = 0; regioncnt < rregions; regioncnt++)
+                    {
+                        LOG((3, "writing data for region with regioncnt = %d", regioncnt));
+
+                        /* Get the start/count arrays for this region. */
+                        for (int i = 0; i < fndims; i++)
+                        {
+                            start[i] = tmp_start[i + regioncnt * fndims];
+                            count[i] = tmp_count[i + regioncnt * fndims];
+                            LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i]));
+                        }
+
+                        /* Process each variable in the buffer. */
+                        for (int nv = 0; nv < nvars; nv++)
+                        {
+                            LOG((3, "writing buffer var %d", nv));
+
+                            /* Get a pointer to the correct part of the buffer. */
+                            bufptr = (void *)((char *)iobuf + tsize * (nv * rlen + loffset));
+
+                            /* If this var has an unlimited dim, set
+                             * the start on that dim to the frame
+                             * value for this variable. */
+                            if (vdesc->record >= 0)
+                            {
+                                if (fndims > 1 && iodesc_ndims < fndims && count[1] > 0)
+                                {
+                                    count[0] = 1;
+                                    start[0] = frame[nv];
+                                }
+                                else if (fndims == iodesc_ndims)
+                                {
+                                    start[0] += vdesc->record;
+                                }
+                            }
+
+                            /* Call the netCDF functions to write the data. */
+                            ierr = nc_put_vara(file->fh, vid[nv], start, count, bufptr);
+
+                            if (ierr)
+                            {
+                                for (int i = 0; i < fndims; i++)
+                                    fprintf(stderr, "vid %d dim %d start %ld count %ld \n", vid[nv], i,
+                                            start[i], count[i]);
+                                return check_netcdf(file, ierr, __FILE__, __LINE__);
+                            }
+                        } /* next var */
+
+                        /* Calculate the total size. */
+                        size_t tsize = 1;
+                        for (int i = 0; i < fndims; i++)
+                            tsize *= count[i];
+
+                        /* Keep track of where we are in the buffer. */
+                        loffset += tsize;
+
+                        LOG((3, " at bottom of loop regioncnt = %d tsize = %d loffset = %d", regioncnt,
+                             tsize, loffset));
+                    } /* next regioncnt */
+                } /* endif (rlen > 0) */
+            } /* next rtask */
+        }
+    }
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:write_darray_multi_nc_serial");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read an array of data from a file to the (parallel) IO library.
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param iodesc a pointer to the defined iodescriptor for the buffer
+ * @param vid the variable id to be read
+ * @param iobuf the buffer to be read into from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @return 0 on success, error code otherwise.
+ * @ingroup PIO_read_darray
+ */
+int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Information about the variable. */
+    int ndims;             /* Number of dims in decomposition. */
+    int fndims;            /* Number of dims for this var in file. */
+    int mpierr;            /* Return code from MPI functions. */
+    int ierr;              /* Return code from netCDF functions. */
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && iodesc && vid <= PIO_MAX_VARS, "invalid input",
+              __FILE__, __LINE__);
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:read_darray_nc");
+#endif
+
+    /* Get the IO system info. */
+    ios = file->iosystem;
+
+    /* Get the variable info. */
+    vdesc = file->varlist + vid;
+
+    /* Get the number of dimensions in the decomposition. */
+    ndims = iodesc->ndims;
+
+    /* Get the number of dims for this var in the file. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Is this a non-record var? */
+    if (fndims == ndims)
+        vdesc->record = -1;
+
+    /* IO procs will actially read the data. */
+    if (ios->ioproc)
+    {
+        io_region *region;
+        size_t start[fndims];
+        size_t count[fndims];
+        size_t tmp_bufsize = 1;
+        void *bufptr;
+        int tsize;
+        int rrlen = 0;
+        PIO_Offset *startlist[iodesc->maxregions];
+        PIO_Offset *countlist[iodesc->maxregions];
+
+        /* buffer is incremented by byte and loffset is in terms of
+           the iodessc->basetype so we need to multiply by the size of
+           the basetype. */
+        region = iodesc->firstregion;
+
+        /* Get the size of the MPI type. */
+        if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* ??? */
+        if (fndims > ndims)
+        {
+            ndims++;
+            if (vdesc->record < 0)
+                vdesc->record = 0;
+        }
+
+        /* For each regions, read the data. */
+        for (int regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++)
+        {
+            tmp_bufsize = 1;
+            if (region == NULL || iodesc->llen == 0)
+            {
+                /* No data for this region. */
+                for (int i = 0; i < fndims; i++)
+                {
+                    start[i] = 0;
+                    count[i] = 0;
+                }
+                bufptr = NULL;
+            }
+            else
+            {
+                /* Get a pointer where we should put the data we read. */
+                if (regioncnt == 0 || region == NULL)
+                    bufptr = iobuf;
+                else
+                    bufptr=(void *)((char *)iobuf + tsize * region->loffset);
+
+                LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset));
+
+                /* Get the start/count arrays. */
+                if (vdesc->record >= 0 && fndims > 1)
+                {
+                    /* This is a record var. The unlimited dimension
+                     * (0) is handled specially. */
+                    start[0] = vdesc->record;
+                    for (int i = 1; i < ndims; i++)
+                    {
+                        start[i] = region->start[i-1];
+                        count[i] = region->count[i-1];
+                    }
+
+                    /* Read one record. */
+                    if (count[1] > 0)
+                        count[0] = 1;
+                }
+                else
+                {
+                    /* Non-time dependent array */
+                    for (int i = 0; i < ndims; i++)
+                    {
+                        start[i] = region->start[i];
+                        count[i] = region->count[i];
+                    }
+                }
+            }
+
+            /* Do the read. */
+            switch (file->iotype)
+            {
+#ifdef _NETCDF4
+            case PIO_IOTYPE_NETCDF4P:
+                ierr = nc_get_vara(file->fh, vid, start, count, bufptr);
+                break;
+#endif
+#ifdef _PNETCDF
+            case PIO_IOTYPE_PNETCDF:
+            {
+                tmp_bufsize = 1;
+                for (int j = 0; j < fndims; j++)
+                    tmp_bufsize *= count[j];
+
+                if (tmp_bufsize > 0)
+                {
+                    startlist[rrlen] = bget(fndims * sizeof(PIO_Offset));
+                    countlist[rrlen] = bget(fndims * sizeof(PIO_Offset));
+
+                    for (int j = 0; j < fndims; j++)
+                    {
+                        startlist[rrlen][j] = start[j];
+                        countlist[rrlen][j] = count[j];
+                    }
+                    rrlen++;
+                }
+
+                /* Is this is the last region to process? */
+                if (regioncnt == iodesc->maxregions - 1)
+                {
+                    /* Read a list of subarrays. */
+                    ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist,
+                                              countlist, iobuf, iodesc->llen, iodesc->basetype);
+
+                    /* Release the start and count arrays. */
+                    for (int i = 0; i < rrlen; i++)
+                    {
+                        brel(startlist[i]);
+                        brel(countlist[i]);
+                    }
+                }
+            }
+            break;
+#endif
+            default:
+                return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+            }
+
+            /* Check return code. */
+            if (ierr)
+                return check_netcdf(file, ierr, __FILE__,__LINE__);
+
+            /* Move to next region. */
+            if (region)
+                region = region->next;
+        } /* next regioncnt */
+    }
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:read_darray_nc");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read an array of data from a file to the (serial) IO library. This
+ * function is only used with netCDF classic and netCDF-4 serial
+ * iotypes.
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param iodesc a pointer to the defined iodescriptor for the buffer
+ * @param vid the variable id to be read.
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize * numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_read_darray
+ */
+int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
+                              void *iobuf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Information about the variable. */
+    int ndims;             /* Number of dims in decomposition. */
+    int fndims;            /* Number of dims for this var in file. */
+    MPI_Status status;
+    int mpierr;  /* Return code from MPI functions. */
+    int ierr;
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && iodesc && vid >= 0 && vid <= PIO_MAX_VARS,
+              "invalid input", __FILE__, __LINE__);
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:read_darray_nc_serial");
+#endif
+    ios = file->iosystem;
+
+    /* Get var info for this var. */
+    vdesc = file->varlist + vid;
+
+    /* Get the number of dims in our decomposition. */
+    ndims = iodesc->ndims;
+
+    /* Get number of dims for this var. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Is this a non-record var? */
+    if (fndims == ndims)
+        vdesc->record = -1;
+
+    if (ios->ioproc)
+    {
+        io_region *region;
+        size_t start[fndims];
+        size_t count[fndims];
+        size_t tmp_start[fndims * iodesc->maxregions];
+        size_t tmp_count[fndims * iodesc->maxregions];
+        size_t tmp_bufsize;
+        void *bufptr;
+        int tsize;
+
+        /* buffer is incremented by byte and loffset is in terms of
+           the iodessc->basetype so we need to multiply by the size of
+           the basetype. */
+        region = iodesc->firstregion;
+
+        /* Get the size of the MPI type. */
+        if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        if (fndims > ndims)
+        {
+            if (vdesc->record < 0)
+                vdesc->record = 0;
+        }
+
+        /* Put together start/count arrays for all regions. */
+        for (int regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++)
+        {
+            if (!region || iodesc->llen == 0)
+            {
+                /* Nothing to write for this region. */
+                for (int i = 0; i < fndims; i++)
+                {
+                    tmp_start[i + regioncnt * fndims] = 0;
+                    tmp_count[i + regioncnt * fndims] = 0;
+                }
+                bufptr = NULL;
+            }
+            else
+            {
+                if (vdesc->record >= 0 && fndims > 1)
+                {
+                    /* This is a record var. Find start for record dims. */
+                    tmp_start[regioncnt * fndims] = vdesc->record;
+
+                    /* Find start/count for all non-record dims. */
+                    for (int i = 1; i < fndims; i++)
+                    {
+                        tmp_start[i + regioncnt * fndims] = region->start[i - 1];
+                        tmp_count[i + regioncnt * fndims] = region->count[i - 1];
+                    }
+
+                    /* Set count for record dimension. */
+                    if (tmp_count[1 + regioncnt * fndims] > 0)
+                        tmp_count[regioncnt * fndims] = 1;
+                }
+                else
+                {
+                    /* Non-time dependent array */
+                    for (int i = 0; i < fndims; i++)
+                    {
+                        tmp_start[i + regioncnt * fndims] = region->start[i];
+                        tmp_count[i + regioncnt * fndims] = region->count[i];
+                    }
+                }
+            }
+
+#if PIO_ENABLE_LOGGING
+            /* Log arrays for debug purposes. */
+            LOG((3, "region = %d", region));
+            for (int i = 0; i < fndims; i++)
+                LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims, tmp_start[i + regioncnt * fndims],
+                     i + regioncnt * fndims, tmp_count[i + regioncnt * fndims]));
+#endif /* PIO_ENABLE_LOGGING */
+
+            /* Move to next region. */
+            if (region)
+                region = region->next;
+        } /* next regioncnt */
+
+        /* IO tasks other than 0 send their starts/counts and data to
+         * IO task 0. */
+        if (ios->io_rank > 0)
+        {
+            if ((mpierr = MPI_Send(&iodesc->llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+            LOG((3, "sent iodesc->llen = %d", iodesc->llen));
+
+            if (iodesc->llen > 0)
+            {
+                if ((mpierr = MPI_Send(&(iodesc->maxregions), 1, MPI_INT, 0,
+                                       ios->num_iotasks + ios->io_rank, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmp_count, iodesc->maxregions * fndims, MPI_OFFSET, 0,
+                                       2 * ios->num_iotasks + ios->io_rank, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmp_start, iodesc->maxregions * fndims, MPI_OFFSET, 0,
+                                       3 * ios->num_iotasks + ios->io_rank, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                LOG((3, "sent iodesc->maxregions = %d tmp_count and tmp_start arrays", iodesc->maxregions));
+
+                if ((mpierr = MPI_Recv(iobuf, iodesc->llen, iodesc->basetype, 0,
+                                       4 * ios->num_iotasks + ios->io_rank, ios->io_comm, &status)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                LOG((3, "received %d elements of data", iodesc->llen));
+            }
+        }
+        else if (ios->io_rank == 0)
+        {
+            /* This is IO task 0. Get starts/counts and data from
+             * other IO tasks. */
+            int maxregions = 0;
+            size_t loffset, regionsize;
+            size_t this_start[fndims * iodesc->maxregions];
+            size_t this_count[fndims * iodesc->maxregions];
+
+            for (int rtask = 1; rtask <= ios->num_iotasks; rtask++)
+            {
+                if (rtask < ios->num_iotasks)
+                {
+                    if ((mpierr = MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status)))
+                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    LOG((3, "received tmp_bufsize = %d", tmp_bufsize));
+
+                    if (tmp_bufsize > 0)
+                    {
+                        if ((mpierr = MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks + rtask,
+                                               ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(this_count, maxregions * fndims, MPI_OFFSET, rtask,
+                                               2 * ios->num_iotasks + rtask, ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(this_start, maxregions * fndims, MPI_OFFSET, rtask,
+                                               3 * ios->num_iotasks + rtask, ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        LOG((3, "received maxregions = %d this_count, this_start arrays ", maxregions));
+                    }
+                }
+                else
+                {
+                    maxregions = iodesc->maxregions;
+                    tmp_bufsize = iodesc->llen;
+                }
+                LOG((3, "maxregions = %d tmp_bufsize = %d", maxregions, tmp_bufsize));
+
+                /* Now get each region of data. */
+                loffset = 0;
+                for (int regioncnt = 0; regioncnt < maxregions; regioncnt++)
+                {
+                    /* Get pointer where data should go. */
+                    bufptr = (void *)((char *)iobuf + tsize * loffset);
+                    regionsize = 1;
+
+                    /* ??? */
+                    if (rtask < ios->num_iotasks)
+                    {
+                        for (int m = 0; m < fndims; m++)
+                        {
+                            start[m] = this_start[m + regioncnt * fndims];
+                            count[m] = this_count[m + regioncnt * fndims];
+                            regionsize *= count[m];
+                        }
+                    }
+                    else
+                    {
+                        for (int m = 0; m < fndims; m++)
+                        {
+                            start[m] = tmp_start[m + regioncnt * fndims];
+                            count[m] = tmp_count[m + regioncnt * fndims];
+                            regionsize *= count[m];
+                        }
+                    }
+                    loffset += regionsize;
+
+                    /* Read the data. */
+                    ierr = nc_get_vara(file->fh, vid, start, count, bufptr);
+
+                    /* Check error code of netCDF call. */
+                    if (ierr)
+                        return check_netcdf(file, ierr, __FILE__, __LINE__);
+                }
+
+                /* The decomposition may not use all of the active io
+                 * tasks. rtask here is the io task rank and
+                 * ios->num_iotasks is the number of iotasks actually
+                 * used in this decomposition. */
+                if (rtask < ios->num_iotasks && tmp_bufsize > 0)
+                    if ((mpierr = MPI_Send(iobuf, tmp_bufsize, iodesc->basetype, rtask,
+                                           4 * ios->num_iotasks + rtask, ios->io_comm)))
+                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+            }
+        }
+    }
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:read_darray_nc_serial");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Flush the output buffer. This is only relevant for files opened
+ * with pnetcdf.
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param force true to force the flushing of the buffer
+ * @param addsize additional size to add to buffer (in bytes)
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
+{
+    int mpierr;  /* Return code from MPI functions. */
+    int ierr = PIO_NOERR;
+
+#ifdef _PNETCDF
+    var_desc_t *vdesc;
+    PIO_Offset usage = 0;
+
+    /* Check inputs. */
+    pioassert(file, "invalid input", __FILE__, __LINE__);
+
+    /* Find out the buffer usage. */
+    if ((ierr = ncmpi_inq_buffer_usage(file->fh, &usage)))
+	return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If we are not forcing a flush, spread the usage to all IO
+     * tasks. */
+    if (!force && file->iosystem->io_comm != MPI_COMM_NULL)
+    {
+        usage += addsize;
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &usage, 1,  MPI_OFFSET,  MPI_MAX,
+                                    file->iosystem->io_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Keep track of the maximum usage. */
+    if (usage > maxusage)
+        maxusage = usage;
+
+    /* If the user forces it, or the buffer has exceeded the size
+     * limit, then flush to disk. */
+    if (force || usage >= pio_buffer_size_limit)
+    {
+        int rcnt;
+        int  maxreq;
+        int reqcnt;
+        maxreq = 0;
+        reqcnt = 0;
+        rcnt = 0;
+        for (int i = 0; i < PIO_MAX_VARS; i++)
+        {
+            vdesc = file->varlist + i;
+            reqcnt += vdesc->nreqs;
+            if (vdesc->nreqs > 0)
+                maxreq = i;
+        }
+        int request[reqcnt];
+        int status[reqcnt];
+
+        for (int i = 0; i <= maxreq; i++)
+        {
+            vdesc = file->varlist + i;
+#ifdef MPIO_ONESIDED
+            /*onesided optimization requires that all of the requests in a wait_all call represent
+              a contiguous block of data in the file */
+            if (rcnt > 0 && (prev_record != vdesc->record || vdesc->nreqs==0))
+            {
+                ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+                rcnt = 0;
+            }
+            prev_record = vdesc->record;
+#endif
+            for (reqcnt = 0; reqcnt < vdesc->nreqs; reqcnt++)
+                request[rcnt++] = max(vdesc->request[reqcnt], NC_REQ_NULL);
+
+            if (vdesc->request != NULL)
+                free(vdesc->request);
+            vdesc->request = NULL;
+            vdesc->nreqs = 0;
+
+#ifdef FLUSH_EVERY_VAR
+            ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+            rcnt = 0;
+#endif
+        }
+
+        if (rcnt > 0)
+            ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+
+        /* Release resources. */
+        for (int i = 0; i < PIO_MAX_VARS; i++)
+        {
+            vdesc = file->varlist + i;
+            if (vdesc->iobuf)
+            {
+                brel(vdesc->iobuf);
+                vdesc->iobuf = NULL;
+            }
+            if (vdesc->fillbuf)
+            {
+                brel(vdesc->fillbuf);
+                vdesc->fillbuf = NULL;
+            }
+        }
+    }
+
+#endif /* _PNETCDF */
+    return ierr;
+}
+
+/**
+ * Print out info about the buffer for debug purposes. This should
+ * only be called when logging is enabled.
+ *
+ * @param ios pointer to the IO system structure
+ * @param collective true if collective report is desired
+ * @ingroup PIO_write_darray
+ */
+void cn_buffer_report(iosystem_desc_t *ios, bool collective)
+{
+    int mpierr;  /* Return code from MPI functions. */
+
+    LOG((2, "cn_buffer_report ios->iossysid = %d collective = %d CN_bpool = %d",
+         ios->iosysid, collective, CN_bpool));
+    if (CN_bpool)
+    {
+        long bget_stats[5];
+        long bget_mins[5];
+        long bget_maxs[5];
+
+        bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4);
+        if (collective)
+        {
+            LOG((3, "cn_buffer_report calling MPI_Reduce ios->comp_comm = %d", ios->comp_comm));
+            if ((mpierr = MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios->comp_comm)))
+                check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "cn_buffer_report calling MPI_Reduce"));
+            if ((mpierr = MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios->comp_comm)))
+                check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if (ios->compmaster == MPI_ROOT)
+            {
+                LOG((1, "Currently allocated buffer space %ld %ld", bget_mins[0], bget_maxs[0]));
+                LOG((1, "Currently available buffer space %ld %ld", bget_mins[1], bget_maxs[1]));
+                LOG((1, "Current largest free block %ld %ld", bget_mins[2], bget_maxs[2]));
+                LOG((1, "Number of successful bget calls %ld %ld", bget_mins[3], bget_maxs[3]));
+                LOG((1, "Number of successful brel calls  %ld %ld", bget_mins[4], bget_maxs[4]));
+            }
+        }
+        else
+        {
+            LOG((1, "Currently allocated buffer space %ld", bget_stats[0]));
+            LOG((1, "Currently available buffer space %ld", bget_stats[1]));
+            LOG((1, "Current largest free block %ld", bget_stats[2]));
+            LOG((1, "Number of successful bget calls %ld", bget_stats[3]));
+            LOG((1, "Number of successful brel calls  %ld", bget_stats[4]));
+        }
+    }
+}
+
+/**
+ * Free the buffer pool. If malloc is used (that is, PIO_USE_MALLOC is
+ * non zero), this function does nothing.
+ *
+ * @param ios pointer to the IO system structure.
+ * @ingroup PIO_write_darray
+ */
+void free_cn_buffer_pool(iosystem_desc_t *ios)
+{
+#if !PIO_USE_MALLOC
+    LOG((2, "free_cn_buffer_pool CN_bpool = %d", CN_bpool));
+    if (CN_bpool)
+    {
+        cn_buffer_report(ios, false);
+        bpoolrelease(CN_bpool);
+        LOG((2, "free_cn_buffer_pool done!"));
+        free(CN_bpool);
+        CN_bpool = NULL;
+    }
+#endif /* !PIO_USE_MALLOC */
+}
+
+/**
+ * Flush the buffer.
+ *
+ * @param ncid identifies the netCDF file
+ * @param wmb May be NULL, in which case function returns.
+ * @param flushtodisk
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk)
+{
+    file_desc_t *file;
+    int ret;
+
+    /* Check input. */
+    pioassert(wmb, "invalid input", __FILE__, __LINE__);
+
+    /* Get the file info (to get error handler). */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "flush_buffer ncid = %d flushtodisk = %d", ncid, flushtodisk));
+
+    /* If there are any variables in this buffer... */
+    if (wmb->validvars > 0)
+    {
+        /* Write any data in the buffer. */
+        ret = PIOc_write_darray_multi(ncid, wmb->vid,  wmb->ioid, wmb->validvars,
+                                      wmb->arraylen, wmb->data, wmb->frame,
+                                      wmb->fillvalue, flushtodisk);
+
+        wmb->validvars = 0;
+
+        /* Release the list of variable IDs. */
+        brel(wmb->vid);
+        wmb->vid = NULL;
+
+        /* Release the data memory. */
+        brel(wmb->data);
+        wmb->data = NULL;
+
+        /* If there is a fill value, release it. */
+        if (wmb->fillvalue)
+            brel(wmb->fillvalue);
+        wmb->fillvalue = NULL;
+
+        /* Release the record number. */
+        if (wmb->frame)
+            brel(wmb->frame);
+        wmb->frame = NULL;
+
+        if (ret)
+            return pio_err(NULL, file, ret, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Compute the maximum aggregate number of bytes.
+ *
+ * @param ios the IO system structure
+ * @param iodesc a pointer to the defined iodescriptor for the
+ * buffer. If NULL, function returns immediately.
+ * @returns 0 for success, error code otherwise.
+ */
+int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int maxbytesoniotask = INT_MAX;
+    int maxbytesoncomputetask = INT_MAX;
+    int maxbytes;
+    int mpierr;  /* Return code from MPI functions. */
+
+    /* Check inputs. */
+    pioassert(iodesc, "invalid input", __FILE__, __LINE__);
+
+    LOG((2, "compute_maxaggregate_bytes iodesc->maxiobuflen = %d iodesc->ndof = %d",
+         iodesc->maxiobuflen, iodesc->ndof));
+
+    if (ios->ioproc && iodesc->maxiobuflen > 0)
+        maxbytesoniotask = pio_buffer_size_limit / iodesc->maxiobuflen;
+
+    if (ios->comp_rank >= 0 && iodesc->ndof > 0)
+        maxbytesoncomputetask = pio_cnbuffer_limit / iodesc->ndof;
+
+    maxbytes = min(maxbytesoniotask, maxbytesoncomputetask);
+    LOG((2, "compute_maxaggregate_bytes maxbytesoniotask = %d maxbytesoncomputetask = %d",
+         maxbytesoniotask, maxbytesoncomputetask));
+
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN,
+                                ios->union_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    iodesc->maxbytes = maxbytes;
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_file.c b/src/externals/pio2/src/clib/pio_file.c
new file mode 100644
index 00000000000..f799e25f57c
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_file.c
@@ -0,0 +1,418 @@
+/**
+ * @file
+ * PIO File Handling
+ */
+#include 
+#include 
+#include 
+
+/* This is the next ncid that will be used when a file is opened or
+   created. We start at 16 so that it will be easy for us to notice
+   that it's not netcdf (starts at 4), pnetcdf (starts at 0) or
+   netCDF-4/HDF5 (starts at 65xxx). */
+int pio_next_ncid = 16;
+
+/**
+ * Open an existing file using PIO library.
+ *
+ * If the open fails, try again as netCDF serial before giving
+ * up. Input parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * Note that the file is opened with default fill mode, NOFILL for
+ * pnetcdf, and FILL for netCDF classic and netCDF-4 files.
+ *
+ * @param iosysid : A defined pio system descriptor (input)
+ * @param ncidp : A pio file descriptor (output)
+ * @param iotype : A pio output format (input)
+ * @param filename : The filename to open
+ * @param mode : The netcdf mode for the open operation
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_openfile
+ */
+int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
+                  int mode)
+{
+    return PIOc_openfile_retry(iosysid, ncidp, iotype, filename, mode, 1);
+}
+
+/**
+ * Open an existing file using PIO library.
+ *
+ * Input parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid A defined pio system descriptor
+ * @param path The filename to open
+ * @param mode The netcdf mode for the open operation
+ * @param ncidp pointer to int where ncid will go
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_openfile
+ */
+int PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
+{
+    int iotype;
+
+    LOG((1, "PIOc_open iosysid = %d path = %s mode = %x", iosysid, path, mode));
+
+    /* Figure out the iotype. */
+    if (mode & NC_NETCDF4)
+    {
+        if (mode & NC_MPIIO || mode & NC_MPIPOSIX)
+            iotype = PIO_IOTYPE_NETCDF4P;
+        else
+            iotype = PIO_IOTYPE_NETCDF4C;
+    }
+    else
+    {
+        if (mode & NC_PNETCDF || mode & NC_MPIIO)
+            iotype = PIO_IOTYPE_PNETCDF;
+        else
+            iotype = PIO_IOTYPE_NETCDF;
+    }
+
+    /* Open the file. If the open fails, do not retry as serial
+     * netCDF. Just return the error code. */
+    return PIOc_openfile_retry(iosysid, ncidp, &iotype, path, mode, 0);
+}
+
+/**
+ * Create a new file using pio. Input parameters are read on comp task
+ * 0 and ignored elsewhere. NOFILL mode will be turned on in all
+ * cases.
+ *
+ * @param iosysid A defined pio system ID, obtained from
+ * PIOc_InitIntercomm() or PIOc_InitAsync().
+ * @param ncidp A pointer that gets the ncid of the newly created
+ * file.
+ * @param iotype A pointer to a pio output format. Must be one of
+ * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or
+ * PIO_IOTYPE_NETCDF4P.
+ * @param filename The filename to create.
+ * @param mode The netcdf mode for the create operation.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_createfile
+ */
+int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
+                    int mode)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ret;               /* Return code from function calls. */
+
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile_int(iosysid, ncidp, iotype, filename, mode)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Set the fill mode to NOFILL. */
+        if ((ret = PIOc_set_fill(*ncidp, NC_NOFILL, NULL)))
+            return ret;
+    }
+
+    return ret;
+}
+
+/**
+ * Open a new file using pio. The default fill mode will be used (FILL
+ * for netCDF and netCDF-4 formats, NOFILL for pnetcdf.) Input
+ * parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid : A defined pio system descriptor (input)
+ * @param cmode : The netcdf mode for the create operation.
+ * @param filename : The filename to open
+ * @param ncidp : A pio file descriptor (output)
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_create
+ */
+int PIOc_create(int iosysid, const char *filename, int cmode, int *ncidp)
+{
+    int iotype;            /* The PIO IO type. */
+
+    /* Figure out the iotype. */
+    if (cmode & NC_NETCDF4)
+    {
+        if (cmode & NC_MPIIO || cmode & NC_MPIPOSIX)
+            iotype = PIO_IOTYPE_NETCDF4P;
+        else
+            iotype = PIO_IOTYPE_NETCDF4C;
+    }
+    else
+    {
+        if (cmode & NC_PNETCDF || cmode & NC_MPIIO)
+            iotype = PIO_IOTYPE_PNETCDF;
+        else
+            iotype = PIO_IOTYPE_NETCDF;
+    }
+
+    return PIOc_createfile_int(iosysid, ncidp, &iotype, filename, cmode);
+}
+
+/**
+ * Close a file previously opened with PIO.
+ *
+ * @param ncid: the file pointer
+ * @returns PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_closefile(int ncid)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_closefile ncid = %d", ncid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Sync changes before closing on all tasks if async is not in
+     * use, but only on non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+        if (file->mode & PIO_WRITE)
+            PIOc_sync(ncid);
+
+    /* If async is in use and this is a comp tasks, then the compmaster
+     * sends a msg to the pio_msg_handler running on the IO master and
+     * waiting for a message. Then broadcast the ncid over the intercomm
+     * to the IO tasks. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_CLOSE_FILE;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        switch (file->iotype)
+        {
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_close(file->fh);
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank == 0)
+                ierr = nc_close(file->fh);
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            if ((file->mode & PIO_WRITE)){
+                ierr = ncmpi_buffer_detach(file->fh);
+            }
+            ierr = ncmpi_close(file->fh);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Delete file from our list of open files. */
+    pio_delete_file_from_list(ncid);
+
+    return ierr;
+}
+
+/**
+ * Delete a file.
+ *
+ * @param iosysid a pio system handle.
+ * @param filename a filename.
+ * @returns PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_deletefile(int iosysid, const char *filename)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+     int msg = PIO_MSG_DELETE_FILE;
+    size_t len;
+
+    LOG((1, "PIOc_deletefile iosysid = %d filename = %s", iosysid, filename));
+
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If async is in use, send message to IO master task. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            if (ios->comp_rank==0)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            len = strlen(filename);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "Bcast len = %d filename = %s", len, filename));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "done hanlding errors mpierr = %d", mpierr));
+    }
+
+    /* If this is an IO task, then call the netCDF function. The
+     * barriers are needed to assure that no task is trying to operate
+     * on the file while it is being deleted. IOTYPE is not known, but
+     * nc_delete() will delete any type of file. */
+    if (ios->ioproc)
+    {
+        mpierr = MPI_Barrier(ios->io_comm);
+
+        if (!mpierr && ios->io_rank == 0)
+             ierr = nc_delete(filename);
+
+        if (!mpierr)
+            mpierr = MPI_Barrier(ios->io_comm);
+    }
+    LOG((2, "PIOc_deletefile ierr = %d", ierr));
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+    return ierr;
+}
+
+/**
+ * PIO interface to nc_sync This routine is called collectively by all
+ * tasks in the communicator ios.union_comm.
+ *
+ * Refer to the  netcdf  documentation.
+ *
+ * @param ncid the ncid of the file to sync.
+ * @returns PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_sync(int ncid)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    wmulti_buffer *wmb, *twmb;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+
+    /* Get the file info from the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, send message to IO master tasks. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SYNC;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (file->mode & PIO_WRITE)
+    {
+        LOG((3, "PIOc_sync checking buffers"));
+
+        /*  cn_buffer_report( *ios, true); */
+        wmb = &file->buffer;
+        while (wmb)
+        {
+            if (wmb->validvars > 0)
+                flush_buffer(ncid, wmb, true);
+            twmb = wmb;
+            wmb = wmb->next;
+            if (twmb == &file->buffer)
+            {
+                twmb->ioid = -1;
+                twmb->next = NULL;
+            }
+            else
+            {
+                brel(twmb);
+            }
+        }
+
+        if (ios->ioproc)
+        {
+            switch(file->iotype)
+            {
+#ifdef _NETCDF4
+            case PIO_IOTYPE_NETCDF4P:
+                ierr = nc_sync(file->fh);
+                break;
+            case PIO_IOTYPE_NETCDF4C:
+#endif
+            case PIO_IOTYPE_NETCDF:
+                if (ios->io_rank == 0)
+                    ierr = nc_sync(file->fh);
+                break;
+#ifdef _PNETCDF
+            case PIO_IOTYPE_PNETCDF:
+                flush_output_buffer(file, true, 0);
+                ierr = ncmpi_sync(file->fh);
+                break;
+#endif
+            default:
+                return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+            }
+        }
+        LOG((2, "PIOc_sync ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+    return ierr;
+}
diff --git a/src/externals/pio2/src/clib/pio_get_nc.c b/src/externals/pio2/src/clib/pio_get_nc.c
new file mode 100644
index 00000000000..e8d54211430
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_get_nc.c
@@ -0,0 +1,1119 @@
+/**
+ * @file
+ * PIO functions to get data (excluding varm functions).
+ *
+ * @author Ed Hartnett
+ * @date  2016
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/**
+ * Get strided, muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_CHAR, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, unsigned char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_schar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, signed char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_BYTE, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_ushort(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const PIO_Offset *stride, unsigned short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_USHORT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_SHORT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_uint(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const PIO_Offset *stride, unsigned int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit int variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_long(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const PIO_Offset *stride, long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_float(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, float *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit floating point
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_double(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const PIO_Offset *stride, double *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 64-bit int
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride,
+                            unsigned long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit int variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride, long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT64, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_text(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_CHAR, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, unsigned char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UBYTE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, signed char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_BYTE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, unsigned short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_USHORT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_SHORT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_long(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, unsigned int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UINT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_int(int ncid, int varid, const PIO_Offset *start,
+                      const PIO_Offset *count, int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_float(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, float *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_FLOAT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_double(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, double *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_DOUBLE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned 64-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, unsigned long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UINT64, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT64, buf);
+}
+
+/**
+ * Get all data of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_text(int ncid, int varid, char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_CHAR, buf);
+}
+
+/**
+ * Get all data of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UBYTE, buf);
+}
+
+/**
+ * Get all data of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_schar(int ncid, int varid, signed char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_BYTE, buf);
+}
+
+/**
+ * Get all data of an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, buf);
+}
+
+/**
+ * Get all data of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_short(int ncid, int varid, short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_SHORT, buf);
+}
+
+/**
+ * Get all data of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT, buf);
+}
+
+/**
+ * Get all data of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_int(int ncid, int varid, int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT, buf);
+}
+
+/**
+ * Get all data of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_long (int ncid, int varid, long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get all data of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_float(int ncid, int varid, float *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_FLOAT, buf);
+}
+
+/**
+ * Get all data of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_double(int ncid, int varid, double *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_DOUBLE, buf);
+}
+
+/**
+ * Get all data of an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT64, buf);
+}
+
+/**
+ * Get all data of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_longlong(int ncid, int varid, long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT64, buf);
+}
+
+/**
+ * Get one value of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_text(int ncid, int varid, const PIO_Offset *index, char *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_CHAR, buf);
+}
+
+/**
+ * Get one value of an unsinged char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_UBYTE, buf);
+}
+
+/**
+ * Get one value of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_schar(int ncid, int varid, const PIO_Offset *index, signed char *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_BYTE, buf);
+}
+
+/**
+ * Get one value of an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_ushort(int ncid, int varid, const PIO_Offset *index, unsigned short *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_USHORT, buf);
+}
+
+/**
+ * Get one value of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_short(int ncid, int varid, const PIO_Offset *index, short *buf)
+{
+    int ret = PIOc_get_var1_tc(ncid, varid, index, NC_SHORT, buf);
+    LOG((1, "PIOc_get_var1_short returned %d", ret));
+    return ret;
+}
+
+/**
+ * Get one value of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_uint(int ncid, int varid, const PIO_Offset *index, unsigned int *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_UINT, buf);
+}
+
+/**
+ * Get one value of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get one value of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_int(int ncid, int varid, const PIO_Offset *index, int *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_INT, buf);
+}
+
+/**
+ * Get one value of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_float(int ncid, int varid, const PIO_Offset *index, float *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_FLOAT, buf);
+}
+
+/**
+ * Get one value of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_DOUBLE, buf);
+}
+
+/**
+ * Get one value of an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index,
+                             unsigned long long *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf);
+}
+
+/**
+ * Get one value of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index,
+                           long long *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf);
+}
+
+/**
+ * Get all data from a variable the same type as the variable in the
+ * file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var(int ncid, int varid, void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_NAT, buf);
+}
+
+/**
+ * Get one value from a variable the same type as the variable in the
+ * file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1(int ncid, int varid, const PIO_Offset *index, void *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_NAT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a variable the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_NAT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a variable of the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const PIO_Offset *stride, void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_NAT, buf);
+}
diff --git a/src/externals/pio2/src/clib/pio_getput_int.c b/src/externals/pio2/src/clib/pio_getput_int.c
new file mode 100644
index 00000000000..32f1968186f
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_getput_int.c
@@ -0,0 +1,1234 @@
+/**
+ * @file
+ * Internal PIO functions to get and put attributes and data
+ * (excluding varm functions).
+ *
+ * @author Ed Hartnett
+ * @date  2016
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/**
+ * Write a netCDF attribute of any type, converting to any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param atttype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
+                    PIO_Offset len, nc_type memtype, const void *op)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    PIO_Offset atttype_len;    /* Length (in bytes) of the att type in file. */
+    PIO_Offset memtype_len;    /* Length of the att data type in memory. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;           /* Return code from function calls. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide some valid parameters. */
+    if (!name || !op || strlen(name) > NC_MAX_NAME || len < 0)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_put_att_tc ncid = %d varid = %d name = %s atttype = %d len = %d memtype = %d",
+         ncid, varid, name, atttype, len, memtype));
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Get the length (in bytes) of the type in file. */
+        if ((ierr = PIOc_inq_type(ncid, atttype, NULL, &atttype_len)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Get the length (in bytes) of the type in memory. */
+        if (memtype == PIO_LONG_INTERNAL)
+            memtype_len = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, memtype, NULL, &memtype_len)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+        LOG((2, "PIOc_put_att atttype_len = %d memtype_len = %d", ncid, atttype_len, memtype_len));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_PUT_ATT;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_OFFSET,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)op, len * memtype_len, MPI_BYTE, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_put_att finished bcast ncid = %d varid = %d namelen = %d name = %s "
+                 "len = %d atttype_len = %d memtype = %d memtype_len = %d", ncid, varid, namelen,
+                 name, len, atttype_len, memtype, memtype_len));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_put_att bcast from comproot = %d atttype_len = %d", ios->comproot,
+             atttype_len, memtype_len));
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            switch(memtype)
+            {
+            case NC_BYTE:
+                ierr = ncmpi_put_att_schar(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_CHAR:
+                ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);
+                break;
+            case NC_SHORT:
+                ierr = ncmpi_put_att_short(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_INT:
+                ierr = ncmpi_put_att_int(file->fh, varid, name, atttype, len, op);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = ncmpi_put_att_long(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_FLOAT:
+                ierr = ncmpi_put_att_float(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_DOUBLE:
+                ierr = ncmpi_put_att_double(file->fh, varid, name, atttype, len, op);
+                break;
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            switch(memtype)
+            {
+            case NC_CHAR:
+                ierr = nc_put_att_text(file->fh, varid, name, len, op);
+                break;
+            case NC_BYTE:
+                ierr = nc_put_att_schar(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_SHORT:
+                ierr = nc_put_att_short(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_INT:
+                ierr = nc_put_att_int(file->fh, varid, name, atttype, len, op);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_put_att_long(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_FLOAT:
+                ierr = nc_put_att_float(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_put_att_double(file->fh, varid, name, atttype, len, op);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_put_att_uchar(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_USHORT:
+                ierr = nc_put_att_ushort(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_UINT:
+                ierr = nc_put_att_uint(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_INT64:
+                LOG((3, "about to call nc_put_att_longlong"));
+                ierr = nc_put_att_longlong(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_UINT64:
+                ierr = nc_put_att_ulonglong(file->fh, varid, name, atttype, len, op);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_put_att_string(file->fh, varid, name, atttype, len, op); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the value of an attribute of any type, converting to any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param memtype the type of the data in memory (if different from
+ * the type of the attribute, the data will be converted to
+ * memtype). The ip pointer points to memory to hold att_len elements
+ * of type memtype.
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip)
+{
+    iosystem_desc_t *ios;   /* Pointer to io system information. */
+    file_desc_t *file;      /* Pointer to file information. */
+    nc_type atttype;        /* The type of the attribute. */
+    PIO_Offset attlen;      /* Number of elements in the attribute array. */
+    PIO_Offset atttype_len; /* Length in bytes of one element of the attribute type. */
+    PIO_Offset memtype_len; /* Length in bytes of one element of the memory type. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+    int ierr;               /* Return code from function calls. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a name and destination pointer. */
+    if (!name || !ip || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_get_att_tc ncid %d varid %d name %s memtype %d",
+         ncid, varid, name, memtype));
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Get the type and length of the attribute. */
+        if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, &attlen)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((2, "atttype = %d attlen = %d", atttype, attlen));
+
+        /* Get the length (in bytes) of the type of the attribute. */
+        if ((ierr = PIOc_inq_type(ncid, atttype, NULL, &atttype_len)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Get the length (in bytes) of the type that the user wants
+         * the data converted to. */
+        if (memtype == PIO_LONG_INTERNAL)
+            memtype_len = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, memtype, NULL, &memtype_len)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+    }
+    LOG((2, "atttype_len = %d memtype_len = %d", atttype_len, memtype_len));
+
+    /* If async is in use, and this is not an IO task, bcast the
+     * parameters and the attribute and type information we fetched. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_ATT;
+            LOG((2, "sending parameters"));
+
+            /* Send the message to IO master. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the function parameters. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "Bcast complete ncid = %d varid = %d namelen = %d name = %s iotype = %d "
+                 "atttype = %d attlen = %d atttype_len = %d", ncid, varid, namelen, name, file->iotype,
+                 atttype, attlen, atttype_len));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "mpi errors handled"));
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        LOG((2, "PIOc_get_att_tc bcast from comproot = %d attlen = %d atttype_len = %d", ios->comproot, attlen, atttype_len));
+        if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_get_att_tc bcast complete attlen = %d atttype_len = %d memtype_len = %d", attlen, atttype_len,
+             memtype_len));
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        LOG((2, "calling pnetcdf/netcdf"));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            switch(memtype)
+            {
+            case NC_BYTE:
+                ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);
+                break;
+            case NC_CHAR:
+                ierr = ncmpi_get_att_text(file->fh, varid, name, ip);
+                break;
+            case NC_SHORT:
+                ierr = ncmpi_get_att_short(file->fh, varid, name, ip);
+                break;
+            case NC_INT:
+                ierr = ncmpi_get_att_int(file->fh, varid, name, ip);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = ncmpi_get_att_long(file->fh, varid, name, ip);
+                break;
+            case NC_FLOAT:
+                ierr = ncmpi_get_att_float(file->fh, varid, name, ip);
+                break;
+            case NC_DOUBLE:
+                ierr = ncmpi_get_att_double(file->fh, varid, name, ip);
+                break;
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            switch(memtype)
+            {
+            case NC_CHAR:
+                ierr = nc_get_att_text(file->fh, varid, name, ip);
+                break;
+            case NC_BYTE:
+                ierr = nc_get_att_schar(file->fh, varid, name, ip);
+                break;
+            case NC_SHORT:
+                ierr = nc_get_att_short(file->fh, varid, name, ip);
+                break;
+            case NC_INT:
+                ierr = nc_get_att_int(file->fh, varid, name, ip);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_get_att_long(file->fh, varid, name, ip);
+                break;
+            case NC_FLOAT:
+                ierr = nc_get_att_float(file->fh, varid, name, ip);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_get_att_double(file->fh, varid, name, ip);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_get_att_uchar(file->fh, varid, name, ip);
+                break;
+            case NC_USHORT:
+                ierr = nc_get_att_ushort(file->fh, varid, name, ip);
+                break;
+            case NC_UINT:
+                ierr = nc_get_att_uint(file->fh, varid, name, ip);
+                break;
+            case NC_INT64:
+                LOG((3, "about to call nc_get_att_longlong"));
+                ierr = nc_get_att_longlong(file->fh, varid, name, ip);
+                break;
+            case NC_UINT64:
+                ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_get_att_string(file->fh, varid, name, ip); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    LOG((2, "ierr = %d", ierr));
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    LOG((2, "bcasting att values attlen = %d memtype_len = %d", attlen, memtype_len));
+    if ((mpierr = MPI_Bcast(ip, (int)attlen * memtype_len, MPI_BYTE, ios->ioroot,
+                            ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "get_att_tc data bcast complete"));
+    return PIO_NOERR;
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_get_vars.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * reading: PIOc_get_vars_text(), PIOc_get_vars_uchar(),
+ * PIOc_get_vars_schar(), PIOc_get_vars_ushort(),
+ * PIOc_get_vars_short(), PIOc_get_vars_uint(), PIOc_get_vars_int(),
+ * PIOc_get_vars_long(), PIOc_get_vars_float(),
+ * PIOc_get_vars_double(), PIOc_get_vars_ulonglong(),
+ * PIOc_get_vars_longlong()
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from the type of the variable being
+ * read from to this type. If NC_NAT then the variable's file type
+ * will be used. Use special PIO_LONG_INTERNAL for _long() functions.
+ * @param buf pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                     const PIO_Offset *stride, nc_type xtype, void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ndims;   /* The number of dimensions in the variable. */
+    PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */
+    PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */
+    char start_present = start ? true : false;
+    char count_present = count ? true : false;
+    char stride_present = stride ? true : false;
+    PIO_Offset *rstart = NULL, *rcount = NULL;
+    nc_type vartype;   /* The type of the var we are reading from. */
+
+    LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d start = %d count = %d "
+         "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a place to put some data. */
+    if (!buf)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Get the type of this var. */
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* If no type was specified, use the var type. */
+        if (xtype == NC_NAT)
+            xtype = vartype;
+
+        /* Handle _long() calls with an special type. */
+        if (xtype == PIO_LONG_INTERNAL)
+            typelen = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+
+        /* Get the number of dims for this var. */
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        PIO_Offset dimlen[ndims];
+
+        /* If no count array was passed, we need to know the dimlens
+         * so we can calculate how many data elements are in the
+         * buf. */
+        if (!count)
+        {
+            int dimid[ndims];
+
+            /* Get the dimids for this var. */
+            if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+            /* Get the length of each dimension. */
+            for (int vd = 0; vd < ndims; vd++)
+                if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd])))
+                    return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+
+        /* Figure out the real start, count, and stride arrays. (The
+         * user may have passed in NULLs.) */
+        /* Allocate memory for these arrays, now that we know ndims. */
+        if (!(rstart = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(rcount = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+        PIO_Offset rstride[ndims];
+        for (int vd = 0; vd < ndims; vd++)
+        {
+            rstart[vd] = start ? start[vd] : 0;
+            rcount[vd] = count ? count[vd] : dimlen[vd];
+            rstride[vd] = stride ? stride[vd] : 1;
+            LOG((3, "rstart[%d] = %d rcount[%d] = %d rstride[%d] = %d", vd,
+                 rstart[vd], vd, rcount[vd], vd, rstride[vd]));
+        }
+
+        /* How many elements in buf? */
+        for (int vd = 0; vd < ndims; vd++)
+            num_elem *= rcount[vd];
+        LOG((2, "PIOc_get_vars_tc num_elem = %d", num_elem));
+
+        /* Free tmp resources. */
+        if (start_present)
+            free(rstart);
+        else
+            start = rstart;
+
+        if (count_present)
+            free(rcount);
+        else
+            count = rcount;
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_VARS;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the function parameters and associated informaiton
+             * to the msg handler. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && stride_present)
+                mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d "
+                 "stride_present = %d xtype = %d num_elem = %d", ncid, varid,
+                 ndims, stride_present, xtype, num_elem));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        LOG((2, "file->iotype = %d xtype = %d file->do_io = %d", file->iotype, xtype, file->do_io));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            ncmpi_begin_indep_data(file->fh);
+
+            /* Only the IO master does the IO, so we are not really
+             * getting parallel IO here. */
+            if (ios->iomaster == MPI_ROOT)
+            {
+                switch(xtype)
+                {
+                case NC_BYTE:
+                    ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_CHAR:
+                    ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_SHORT:
+                    ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_INT:
+                    ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride, buf);
+                    break;
+                case PIO_LONG_INTERNAL:
+                    ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_FLOAT:
+                    ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_DOUBLE:
+                    ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);
+                    break;
+                default:
+                    return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+                }
+            };
+            ncmpi_end_indep_data(file->fh);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            switch(xtype)
+            {
+            case NC_BYTE:
+                ierr = nc_get_vars_schar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_CHAR:
+                ierr = nc_get_vars_text(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_SHORT:
+                ierr = nc_get_vars_short(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT:
+                ierr = nc_get_vars_int(file->fh, varid, (size_t *)start, (size_t *)count,
+                                       (ptrdiff_t *)stride, buf);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_get_vars_long(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_FLOAT:
+                ierr = nc_get_vars_float(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_get_vars_double(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_get_vars_uchar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_USHORT:
+                ierr = nc_get_vars_ushort(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT:
+                ierr = nc_get_vars_uint(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT64:
+                LOG((3, "about to call nc_get_vars_longlong"));
+                ierr = nc_get_vars_longlong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                            (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT64:
+                ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                             (ptrdiff_t *)stride, buf);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_get_vars_string(file->fh, varid, (size_t *)start, (size_t *)count, */
+                /*                                (ptrdiff_t *)stride, (void *)buf); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+    }
+
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Free tmp start/count allocated to account for NULL start/counts */
+        if (!start_present)
+            free(rstart);
+        if (!count_present)
+            free(rcount);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Send the data. */
+    LOG((2, "PIOc_get_vars_tc bcasting data num_elem = %d typelen = %d ios->ioroot = %d", num_elem,
+         typelen, ios->ioroot));
+    if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "PIOc_get_vars_tc bcasting data complete"));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get one value of a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param xtype the netcdf type of the variable.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                     void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;   /* The number of dimensions in the variable. */
+    int ierr;    /* Return code from function calls. */
+
+    /* Find the info about this file. We need this for error handling. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Find the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Set up count array. */
+    PIO_Offset count[ndims];
+    for (int c = 0; c < ndims; c++)
+        count[c] = 1;
+
+    return PIOc_get_vars_tc(ncid, varid, index, count, NULL, xtype, buf);
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_put_vars.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * writing: PIOc_put_vars_text(), PIOc_put_vars_uchar(),
+ * PIOc_put_vars_schar(), PIOc_put_vars_ushort(),
+ * PIOc_put_vars_short(), PIOc_put_vars_uint(), PIOc_put_vars_int(),
+ * PIOc_put_vars_long(), PIOc_put_vars_float(),
+ * PIOc_put_vars_longlong(), PIOc_put_vars_double(),
+ * PIOc_put_vars_ulonglong().
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from this type to the type of the
+ * variable being written to. If NC_NAT then the variable's file type
+ * will be used. Use special PIO_LONG_INTERNAL for _long() functions.
+ * @param buf pointer to the data to be written.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                     const PIO_Offset *stride, nc_type xtype, const void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims; /* The number of dimensions in the variable. */
+    PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */
+    PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */
+    char start_present = start ? true : false; /* Is start non-NULL? */
+    char count_present = count ? true : false; /* Is count non-NULL? */
+    char stride_present = stride ? true : false; /* Is stride non-NULL? */
+    PIO_Offset *rstart, *rcount, *rstride;
+    var_desc_t *vdesc;
+    int *request;
+    nc_type vartype;   /* The type of the var we are reading from. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;          /* Return code from function calls. */
+
+    LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start = %d count = %d "
+         "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype));
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a place to put some data. */
+    if (!buf)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Get the type of this var. */
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* If no type was specified, use the var type. */
+        if (xtype == NC_NAT)
+            xtype = vartype;
+
+        /* Get the number of dims for this var. */
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Get the length of the data type. */
+        if (xtype == PIO_LONG_INTERNAL)
+            typelen = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+
+        LOG((2, "ndims = %d typelen = %d", ndims, typelen));
+
+        PIO_Offset dimlen[ndims];
+
+        /* If no count array was passed, we need to know the dimlens
+         * so we can calculate how many data elements are in the
+         * buf. */
+        if (!count)
+        {
+            int dimid[ndims];
+
+            /* Get the dimids for this var. */
+            if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+            /* Get the length of each dimension. */
+            for (int vd = 0; vd < ndims; vd++)
+            {
+                if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd])))
+                    return check_netcdf(file, ierr, __FILE__, __LINE__);
+                LOG((3, "dimlen[%d] = %d", vd, dimlen[vd]));
+            }
+        }
+
+        /* Allocate memory for these arrays, now that we know ndims. */
+        if (!(rstart = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(rcount = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(rstride = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Figure out the real start, count, and stride arrays. (The
+         * user may have passed in NULLs.) */
+        for (int vd = 0; vd < ndims; vd++)
+        {
+            rstart[vd] = start ? start[vd] : 0;
+            rcount[vd] = count ? count[vd] : dimlen[vd];
+            rstride[vd] = stride ? stride[vd] : 1;
+            LOG((3, "rstart[%d] = %d rcount[%d] = %d rstride[%d] = %d", vd,
+                 rstart[vd], vd, rcount[vd], vd, rstride[vd]));
+        }
+
+        /* How many elements in buf? */
+        for (int vd = 0; vd < ndims; vd++)
+            num_elem *= rcount[vd];
+        LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem));
+
+        /* Free tmp resources. */
+        if (start_present)
+            free(rstart);
+        else
+            start = rstart;
+
+        if (count_present)
+            free(rcount);
+        else
+            count = rcount;
+
+        /* Only PNETCDF requires a non-NULL stride, realocate it later if needed */
+        free(rstride);
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_PUT_VARS;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the function parameters and associated informaiton
+             * to the msg handler. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && stride_present)
+                mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d ndims = %d start_present = %d "
+                 "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid,
+                 ndims, start_present, count_present, stride_present, xtype, num_elem));
+
+            /* Send the data. */
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmaster,
+                                   ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr));
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        LOG((2, "PIOc_put_vars_tc bcast from comproot"));
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_put_vars_tc complete bcast from comproot ndims = %d", ndims));
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            PIO_Offset *fake_stride;
+
+            if (!stride_present)
+            {
+                LOG((2, "stride not present"));
+                if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset))))
+                    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+                for (int d = 0; d < ndims; d++)
+                    fake_stride[d] = 1;
+            }
+            else
+                fake_stride = (PIO_Offset *)stride;
+
+            LOG((2, "PIOc_put_vars_tc calling pnetcdf function"));
+            vdesc = file->varlist + varid;
+            if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0)
+                if (!(vdesc->request = realloc(vdesc->request,
+                                               sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK))))
+                    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+            request = vdesc->request + vdesc->nreqs;
+            LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request));
+
+            /* Only the IO master actually does the call. */
+            if (ios->iomaster == MPI_ROOT)
+            {
+                switch(xtype)
+                {
+                case NC_BYTE:
+                    ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                case NC_CHAR:
+                    ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                case NC_SHORT:
+                    ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                case NC_INT:
+                    ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                case PIO_LONG_INTERNAL:
+                    ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                case NC_FLOAT:
+                    ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                case NC_DOUBLE:
+                    ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request);
+                    break;
+                default:
+                    return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+                }
+                LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call, ierr=%d", ierr));
+            }
+            else
+                *request = PIO_REQ_NULL;
+
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            LOG((2, "PIOc_put_vars_tc flushed output buffer"));
+
+            /* Free malloced resources. */
+            if (!stride_present)
+                free(fake_stride);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            LOG((2, "PIOc_put_vars_tc calling netcdf function file->iotype = %d",
+                 file->iotype));
+            switch(xtype)
+            {
+            case NC_BYTE:
+                ierr = nc_put_vars_schar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_CHAR:
+                ierr = nc_put_vars_text(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_SHORT:
+                ierr = nc_put_vars_short(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT:
+                ierr = nc_put_vars_int(file->fh, varid, (size_t *)start, (size_t *)count,
+                                       (ptrdiff_t *)stride, buf);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_put_vars_long(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_FLOAT:
+                ierr = nc_put_vars_float(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_put_vars_double(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_put_vars_uchar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_USHORT:
+                ierr = nc_put_vars_ushort(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT:
+                ierr = nc_put_vars_uint(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT64:
+                ierr = nc_put_vars_longlong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                            (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT64:
+                ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                             (ptrdiff_t *)stride, buf);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_put_vars_string(file->fh, varid, (size_t *)start, (size_t *)count, */
+                /*                                (ptrdiff_t *)stride, (void *)buf); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+            LOG((2, "PIOc_put_vars_tc io_rank 0 done with netcdf call, ierr=%d", ierr));
+        }
+    }
+
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Free tmp start/count allocated to account for NULL start/counts */
+        if (!start_present)
+            free(rstart);
+        if (!count_present)
+            free(rcount);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((2, "PIOc_put_vars_tc bcast netcdf return code %d complete", ierr));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_put_var1 calls.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * writing: PIOc_put_var1_text(), PIOc_put_var1_uchar(),
+ * PIOc_put_var1_schar(), PIOc_put_var1_ushort(),
+ * PIOc_put_var1_short(), PIOc_put_var1_uint(), PIOc_put_var1_int(),
+ * PIOc_put_var1_long(), PIOc_put_var1_float(),
+ * PIOc_put_var1_longlong(), PIOc_put_var1_double(),
+ * PIOc_put_var1_ulonglong().
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from this type to the type of the
+ * variable being written to.
+ * @param op pointer to the data to be written.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                     const void *op)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;   /* The number of dimensions in the variable. */
+    int ierr;    /* Return code from function calls. */
+
+    /* Find the info about this file. We need this for error handling. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Find the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Set up count array. */
+    PIO_Offset count[ndims];
+    for (int c = 0; c < ndims; c++)
+        count[c] = 1;
+
+    return PIOc_put_vars_tc(ncid, varid, index, count, NULL, xtype, op);
+}
diff --git a/src/externals/pio2/src/clib/pio_internal.h b/src/externals/pio2/src/clib/pio_internal.h
new file mode 100644
index 00000000000..e1c891c761e
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_internal.h
@@ -0,0 +1,516 @@
+/**
+ * @file
+ * Private headers and defines for the PIO C interface.
+ * @author Jim Edwards
+ * @date  2014
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#ifndef __PIO_INTERNAL__
+#define __PIO_INTERNAL__
+
+#include 
+
+/* It seems that some versions of openmpi fail to define
+ * MPI_OFFSET. */
+#ifdef OMPI_OFFSET_DATATYPE
+#ifndef MPI_OFFSET
+#define MPI_OFFSET OMPI_OFFSET_DATATYPE
+#endif
+#endif
+#ifndef MPI_Offset
+#define MPI_Offset long long
+#endif
+
+#if defined(MPT_VERSION) || defined(OPEN_MPI)
+/* Some MPI implementations do not allow passing MPI_DATATYPE_NULL to comm functions
+ * even though the send or recv length is 0, in these cases we use MPI_CHAR */
+#define PIO_DATATYPE_NULL MPI_CHAR
+#else
+#define PIO_DATATYPE_NULL MPI_DATATYPE_NULL
+#endif
+
+#include 
+#include 
+#include 
+#ifdef TIMING
+#include 
+#endif
+#include 
+
+#if PIO_ENABLE_LOGGING
+void pio_log(int severity, const char *fmt, ...);
+#define LOG(e) pio_log e
+#else
+#define LOG(e)
+#endif /* PIO_ENABLE_LOGGING */
+
+#define max(a,b)                                \
+    ({ __typeof__ (a) _a = (a);                 \
+        __typeof__ (b) _b = (b);                \
+        _a > _b ? _a : _b; })
+
+#define min(a,b)                                \
+    ({ __typeof__ (a) _a = (a);                 \
+        __typeof__ (b) _b = (b);                \
+        _a < _b ? _a : _b; })
+
+#define MAX_GATHER_BLOCK_SIZE 0
+#define PIO_REQUEST_ALLOC_CHUNK 16
+
+/** This is needed to handle _long() functions. It may not be used as
+ * a data type when creating attributes or varaibles, it is only used
+ * internally. */
+#define PIO_LONG_INTERNAL 13
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+    extern PIO_Offset pio_buffer_size_limit;
+
+    /** Used to sort map points in the subset rearranger. */
+    typedef struct mapsort
+    {
+        int rfrom;
+        PIO_Offset soffset;
+        PIO_Offset iomap;
+    } mapsort;
+
+    /** swapm defaults. */
+    typedef struct pio_swapm_defaults
+    {
+        int nreqs;
+        bool handshake;
+        bool isend;
+    } pio_swapm_defaults;
+
+    /* Handle an error in the PIO library. */
+    int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
+                int line);
+
+    /* For async cases, this runs on IO tasks and listens for messages. */
+    int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
+                         MPI_Comm io_comm);
+
+    void pio_get_env(void);
+    int  pio_add_to_iodesc_list(io_desc_t *iodesc);
+    io_desc_t *pio_get_iodesc_from_id(int ioid);
+    int pio_delete_iodesc_from_list(int ioid);
+    int pio_num_iosystem(int *niosysid);
+
+    int pio_get_file(int ncid, file_desc_t **filep);
+    int pio_delete_file_from_list(int ncid);
+    void pio_add_to_file_list(file_desc_t *file);
+    void pio_push_request(file_desc_t *file, int request);
+
+    /* Create a file (internal function). */
+    int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, int mode);
+
+    /* Open a file with optional retry as netCDF-classic if first
+     * iotype does not work. */
+    int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype,
+                            const char *filename, int mode, int retry);
+
+    iosystem_desc_t *pio_get_iosystem_from_id(int iosysid);
+    int pio_add_to_iosystem_list(iosystem_desc_t *ios);
+
+    /* Check the return code from a netCDF call. */
+    int check_netcdf(file_desc_t *file, int status, const char *fname, int line);
+
+    /* Check the return code from a netCDF call, with ios pointer. */
+    int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
+                      const char *fname, int line);
+
+    /* Find the MPI type that matches a PIO type. */
+    int find_mpi_type(int pio_type, MPI_Datatype *mpi_type);
+
+    /* Check whether an IO type is valid for this build. */
+    int iotype_is_valid(int iotype);
+
+    /* Print error message and abort. */
+    void piodie(const char *msg, const char *fname, int line);
+
+    /* Assert that an expression is true. */
+    void pioassert(bool exp, const char *msg, const char *fname, int line);
+
+    int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_procs,
+                          int myiorank, PIO_Offset *start, PIO_Offset *kount);
+
+    /* Check return from MPI function and print error message. */
+    void CheckMPIReturn(int ierr, const char *file, int line);
+
+    /* Like MPI_Alltoallw(), but with flow control. */
+    int pio_swapm(void *sndbuf, int *sndlths, int *sdispls, MPI_Datatype *stypes,
+                  void *rcvbuf, int *rcvlths, int *rdispls, MPI_Datatype *rtypes,
+                  MPI_Comm comm, bool handshake, bool isend, int max_requests);
+
+    long long lgcd_array(int nain, long long* ain);
+
+    void PIO_Offset_size(MPI_Datatype *dtype, int *tsize);
+    PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in);
+
+    /* Initialize the rearranger options. */
+    void init_rearr_opts(iosystem_desc_t *iosys);
+
+    /* Compare sets of rearranger options. */
+    bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts);
+
+    /* Reset rearranger opts in iosystem to valid values. */
+    void check_and_reset_rearr_opts(iosystem_desc_t *ios);
+
+    /* Compare rearranger flow control options. */
+    bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt,
+                                const rearr_comm_fc_opt_t *exp_opt);
+
+    /* Create a subset rearranger. */
+    int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, const int *gsize,
+                                int ndim, io_desc_t *iodesc);
+
+
+    /* Create a box rearranger. */
+    int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap, const int *gsize,
+                             int ndim, io_desc_t *iodesc);
+
+
+    /* Move data from IO tasks to compute tasks. */
+    int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf);
+
+    /* Move data from compute tasks to IO tasks. */
+    int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, int nvars);
+
+    /* Allocate and initialize storage for decomposition information. */
+    int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims, io_desc_t **iodesc);
+    void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize);
+    int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc);
+    io_region *alloc_region(int ndims);
+    int pio_delete_iosystem_from_list(int piosysid);
+
+    /* Find greatest commond divisor. */
+    int gcd(int a, int b);
+
+    /* Find greatest commond divisor for long long. */
+    long long lgcd (long long a, long long b );
+
+    /* Find greatest commond divisor in an array. */
+    int gcd_array(int nain, int *ain);
+
+    void free_region_list(io_region *top);
+
+    /* Convert a global coordinate value into a local array index. */
+    PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count);
+
+    int ceil2(int i);
+    int pair(int np, int p, int k);
+
+    /* Create MPI datatypes used for comp2io and io2comp data transfers. */
+    int define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    /* Create the derived MPI datatypes used for comp2io and io2comp
+     * transfers. */
+    int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, const PIO_Offset *mindex,
+                             const int *mcount, int *mfrom, MPI_Datatype *mtype);
+    int compare_offsets(const void *a, const void *b) ;
+
+    /* Print a trace statement, for debugging. */
+    void print_trace (FILE *fp);
+
+    void cn_buffer_report(iosystem_desc_t *ios, bool collective);
+
+    /* Initialize the compute buffer. */
+    int compute_buffer_init(iosystem_desc_t *ios);
+
+    void free_cn_buffer_pool(iosystem_desc_t *ios);
+
+    /* Flush PIO's data buffer. */
+    int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk);
+
+    int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    /* Announce a memory error with bget memory, and die. */
+    void piomemerror(iosystem_desc_t *ios, size_t req, char *fname, int line);
+
+    /* Check the return code from an MPI function call. */
+    int check_mpi(file_desc_t *file, int mpierr, const char *filename, int line);
+
+    /* Check the return code from an MPI function call. */
+    int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr, const char *filename,
+                   int line);
+
+    /* Darray support functions. */
+
+    /* Write aggregated arrays to file using parallel I/O (netCDF-4 parallel/pnetcdf) */
+    int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims,
+                                  MPI_Datatype basetype, int maxregions, io_region *firstregion,
+                                  PIO_Offset llen, int num_aiotasks, void *iobuf,
+                                  const int *frame);
+
+    /* Write aggregated arrays to file using serial I/O (netCDF-3/netCDF-4 serial) */
+    int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims,
+                                         MPI_Datatype basetype, int maxregions, io_region *firstregion,
+                                         PIO_Offset llen, int num_aiotasks, void *iobuf,
+                                         const int *frame);
+
+    int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf);
+    int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf);
+
+    /* Read atts with type conversion. */
+    int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip);
+
+    /* Write atts with type conversion. */
+    int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
+                        PIO_Offset len, nc_type memtype, const void *op);
+
+    /* Generalized get functions. */
+    int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, nc_type xtype, void *buf);
+    int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                         void *buf);
+
+    /* Generalized put functions. */
+    int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, nc_type xtype, const void *buf);
+    int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                         const void *op);
+
+    /* An internal replacement for a function pnetcdf does not
+     * have. */
+    int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
+                              PIO_Offset *sizep);
+
+    /* Handle end and re-defs. */
+    int pioc_change_def(int ncid, int is_enddef);
+
+    /* Initialize and finalize logging. */
+    void pio_init_logging(void);
+    void pio_finalize_logging(void );
+
+    /* Write a netCDF decomp file. */
+    int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int ndims, int *global_dimlen,
+                                 int num_tasks, int *task_maplen, int *map, const char *title,
+                                 const char *history, int fortran_order);
+
+    /* Read a netCDF decomp file. */
+    int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen,
+                                int *num_tasks, int **task_maplen, int *max_maplen, int **map, char *title,
+                                char *history, char *source, char *version, int *fortran_order);
+
+#if defined(__cplusplus)
+}
+#endif
+
+/** These are the messages that can be sent over the intercomm when
+ * async is being used. */
+enum PIO_MSG
+{
+    PIO_MSG_OPEN_FILE,
+    PIO_MSG_CREATE_FILE,
+    PIO_MSG_INQ_ATT,
+    PIO_MSG_INQ_FORMAT,
+    PIO_MSG_INQ_VARID,
+    PIO_MSG_DEF_VAR,
+    PIO_MSG_INQ_VAR,
+    PIO_MSG_PUT_ATT_DOUBLE,
+    PIO_MSG_PUT_ATT_INT,
+    PIO_MSG_RENAME_ATT,
+    PIO_MSG_DEL_ATT,
+    PIO_MSG_INQ,
+    PIO_MSG_GET_ATT_TEXT,
+    PIO_MSG_GET_ATT_SHORT,
+    PIO_MSG_PUT_ATT_LONG,
+    PIO_MSG_REDEF,
+    PIO_MSG_SET_FILL,
+    PIO_MSG_ENDDEF,
+    PIO_MSG_RENAME_VAR,
+    PIO_MSG_PUT_ATT_SHORT,
+    PIO_MSG_PUT_ATT_TEXT,
+    PIO_MSG_INQ_ATTNAME,
+    PIO_MSG_GET_ATT_ULONGLONG,
+    PIO_MSG_GET_ATT_USHORT,
+    PIO_MSG_PUT_ATT_ULONGLONG,
+    PIO_MSG_GET_ATT_UINT,
+    PIO_MSG_GET_ATT_LONGLONG,
+    PIO_MSG_PUT_ATT_SCHAR,
+    PIO_MSG_PUT_ATT_FLOAT,
+    PIO_MSG_RENAME_DIM,
+    PIO_MSG_GET_ATT_LONG,
+    PIO_MSG_INQ_DIM,
+    PIO_MSG_INQ_DIMID,
+    PIO_MSG_PUT_ATT_USHORT,
+    PIO_MSG_GET_ATT_FLOAT,
+    PIO_MSG_SYNC,
+    PIO_MSG_PUT_ATT_LONGLONG,
+    PIO_MSG_PUT_ATT_UINT,
+    PIO_MSG_GET_ATT_SCHAR,
+    PIO_MSG_INQ_ATTID,
+    PIO_MSG_DEF_DIM,
+    PIO_MSG_GET_ATT_INT,
+    PIO_MSG_GET_ATT_DOUBLE,
+    PIO_MSG_PUT_ATT_UCHAR,
+    PIO_MSG_GET_ATT_UCHAR,
+    PIO_MSG_PUT_VARS_UCHAR,
+    PIO_MSG_GET_VAR1_SCHAR,
+    PIO_MSG_GET_VARS_ULONGLONG,
+    PIO_MSG_GET_VARM_UCHAR,
+    PIO_MSG_GET_VARM_SCHAR,
+    PIO_MSG_GET_VARS_SHORT,
+    PIO_MSG_GET_VAR_DOUBLE,
+    PIO_MSG_GET_VARA_DOUBLE,
+    PIO_MSG_GET_VAR_INT,
+    PIO_MSG_GET_VAR_USHORT,
+    PIO_MSG_PUT_VARS_USHORT,
+    PIO_MSG_GET_VARA_TEXT,
+    PIO_MSG_PUT_VARS_ULONGLONG,
+    PIO_MSG_GET_VARA_INT,
+    PIO_MSG_PUT_VARM,
+    PIO_MSG_GET_VAR1_FLOAT,
+    PIO_MSG_GET_VAR1_SHORT,
+    PIO_MSG_GET_VARS_INT,
+    PIO_MSG_PUT_VARS_UINT,
+    PIO_MSG_GET_VAR_TEXT,
+    PIO_MSG_GET_VARM_DOUBLE,
+    PIO_MSG_PUT_VARM_UCHAR,
+    PIO_MSG_PUT_VAR_USHORT,
+    PIO_MSG_GET_VARS_SCHAR,
+    PIO_MSG_GET_VARA_USHORT,
+    PIO_MSG_PUT_VAR1_LONGLONG,
+    PIO_MSG_PUT_VARA_UCHAR,
+    PIO_MSG_PUT_VARM_SHORT,
+    PIO_MSG_PUT_VAR1_LONG,
+    PIO_MSG_PUT_VARS_LONG,
+    PIO_MSG_GET_VAR1_USHORT,
+    PIO_MSG_PUT_VAR_SHORT,
+    PIO_MSG_PUT_VARA_INT,
+    PIO_MSG_GET_VAR_FLOAT,
+    PIO_MSG_PUT_VAR1_USHORT,
+    PIO_MSG_PUT_VARA_TEXT,
+    PIO_MSG_PUT_VARM_TEXT,
+    PIO_MSG_GET_VARS_UCHAR,
+    PIO_MSG_GET_VAR,
+    PIO_MSG_PUT_VARM_USHORT,
+    PIO_MSG_GET_VAR1_LONGLONG,
+    PIO_MSG_GET_VARS_USHORT,
+    PIO_MSG_GET_VAR_LONG,
+    PIO_MSG_GET_VAR1_DOUBLE,
+    PIO_MSG_PUT_VAR_ULONGLONG,
+    PIO_MSG_PUT_VAR_INT,
+    PIO_MSG_GET_VARA_UINT,
+    PIO_MSG_PUT_VAR_LONGLONG,
+    PIO_MSG_GET_VARS_LONGLONG,
+    PIO_MSG_PUT_VAR_SCHAR,
+    PIO_MSG_PUT_VAR_UINT,
+    PIO_MSG_PUT_VAR,
+    PIO_MSG_PUT_VARA_USHORT,
+    PIO_MSG_GET_VAR_LONGLONG,
+    PIO_MSG_GET_VARA_SHORT,
+    PIO_MSG_PUT_VARS_SHORT,
+    PIO_MSG_PUT_VARA_UINT,
+    PIO_MSG_PUT_VARA_SCHAR,
+    PIO_MSG_PUT_VARM_ULONGLONG,
+    PIO_MSG_PUT_VAR1_UCHAR,
+    PIO_MSG_PUT_VARM_INT,
+    PIO_MSG_PUT_VARS_SCHAR,
+    PIO_MSG_GET_VARA_LONG,
+    PIO_MSG_PUT_VAR1,
+    PIO_MSG_GET_VAR1_INT,
+    PIO_MSG_GET_VAR1_ULONGLONG,
+    PIO_MSG_GET_VAR_UCHAR,
+    PIO_MSG_PUT_VARA_FLOAT,
+    PIO_MSG_GET_VARA_UCHAR,
+    PIO_MSG_GET_VARS_FLOAT,
+    PIO_MSG_PUT_VAR1_FLOAT,
+    PIO_MSG_PUT_VARM_FLOAT,
+    PIO_MSG_PUT_VAR1_TEXT,
+    PIO_MSG_PUT_VARS_TEXT,
+    PIO_MSG_PUT_VARM_LONG,
+    PIO_MSG_GET_VARS_LONG,
+    PIO_MSG_PUT_VARS_DOUBLE,
+    PIO_MSG_GET_VAR1,
+    PIO_MSG_GET_VAR_UINT,
+    PIO_MSG_PUT_VARA_LONGLONG,
+    PIO_MSG_GET_VARA,
+    PIO_MSG_PUT_VAR_DOUBLE,
+    PIO_MSG_GET_VARA_SCHAR,
+    PIO_MSG_PUT_VAR_FLOAT,
+    PIO_MSG_GET_VAR1_UINT,
+    PIO_MSG_GET_VARS_UINT,
+    PIO_MSG_PUT_VAR1_ULONGLONG,
+    PIO_MSG_PUT_VARM_UINT,
+    PIO_MSG_PUT_VAR1_UINT,
+    PIO_MSG_PUT_VAR1_INT,
+    PIO_MSG_GET_VARA_FLOAT,
+    PIO_MSG_GET_VARM_TEXT,
+    PIO_MSG_PUT_VARS_FLOAT,
+    PIO_MSG_GET_VAR1_TEXT,
+    PIO_MSG_PUT_VARA_SHORT,
+    PIO_MSG_PUT_VAR1_SCHAR,
+    PIO_MSG_PUT_VARA_ULONGLONG,
+    PIO_MSG_PUT_VARM_DOUBLE,
+    PIO_MSG_GET_VARM_INT,
+    PIO_MSG_PUT_VARA,
+    PIO_MSG_PUT_VARA_LONG,
+    PIO_MSG_GET_VARM_UINT,
+    PIO_MSG_GET_VARM,
+    PIO_MSG_PUT_VAR1_DOUBLE,
+    PIO_MSG_GET_VARS_DOUBLE,
+    PIO_MSG_GET_VARA_LONGLONG,
+    PIO_MSG_GET_VAR_ULONGLONG,
+    PIO_MSG_PUT_VARM_SCHAR,
+    PIO_MSG_GET_VARA_ULONGLONG,
+    PIO_MSG_GET_VAR_SHORT,
+    PIO_MSG_GET_VARM_FLOAT,
+    PIO_MSG_PUT_VAR_TEXT,
+    PIO_MSG_PUT_VARS_INT,
+    PIO_MSG_GET_VAR1_LONG,
+    PIO_MSG_GET_VARM_LONG,
+    PIO_MSG_GET_VARM_USHORT,
+    PIO_MSG_PUT_VAR1_SHORT,
+    PIO_MSG_PUT_VARS_LONGLONG,
+    PIO_MSG_GET_VARM_LONGLONG,
+    PIO_MSG_GET_VARS_TEXT,
+    PIO_MSG_PUT_VARA_DOUBLE,
+    PIO_MSG_PUT_VARS,
+    PIO_MSG_PUT_VAR_UCHAR,
+    PIO_MSG_GET_VAR1_UCHAR,
+    PIO_MSG_PUT_VAR_LONG,
+    PIO_MSG_GET_VARS,
+    PIO_MSG_GET_VARM_SHORT,
+    PIO_MSG_GET_VARM_ULONGLONG,
+    PIO_MSG_PUT_VARM_LONGLONG,
+    PIO_MSG_GET_VAR_SCHAR,
+    PIO_MSG_GET_ATT_UBYTE,
+    PIO_MSG_PUT_ATT_STRING,
+    PIO_MSG_GET_ATT_STRING,
+    PIO_MSG_PUT_ATT_UBYTE,
+    PIO_MSG_INQ_VAR_FILL,
+    PIO_MSG_DEF_VAR_FILL,
+    PIO_MSG_DEF_VAR_DEFLATE,
+    PIO_MSG_INQ_VAR_DEFLATE,
+    PIO_MSG_INQ_VAR_SZIP,
+    PIO_MSG_DEF_VAR_FLETCHER32,
+    PIO_MSG_INQ_VAR_FLETCHER32,
+    PIO_MSG_DEF_VAR_CHUNKING,
+    PIO_MSG_INQ_VAR_CHUNKING,
+    PIO_MSG_DEF_VAR_ENDIAN,
+    PIO_MSG_INQ_VAR_ENDIAN,
+    PIO_MSG_SET_CHUNK_CACHE,
+    PIO_MSG_GET_CHUNK_CACHE,
+    PIO_MSG_SET_VAR_CHUNK_CACHE,
+    PIO_MSG_GET_VAR_CHUNK_CACHE,
+    PIO_MSG_INITDECOMP_DOF,
+    PIO_MSG_WRITEDARRAY,
+    PIO_MSG_READDARRAY,
+    PIO_MSG_SETERRORHANDLING,
+    PIO_MSG_FREEDECOMP,
+    PIO_MSG_CLOSE_FILE,
+    PIO_MSG_DELETE_FILE,
+    PIO_MSG_EXIT,
+    PIO_MSG_GET_ATT,
+    PIO_MSG_PUT_ATT,
+    PIO_MSG_INQ_TYPE
+};
+
+#endif /* __PIO_INTERNAL__ */
diff --git a/src/externals/pio2/src/clib/pio_lists.c b/src/externals/pio2/src/clib/pio_lists.c
new file mode 100644
index 00000000000..72e774715bc
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_lists.c
@@ -0,0 +1,299 @@
+/**
+ * @file
+ * PIO list functions.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+static io_desc_t *pio_iodesc_list=NULL;
+static io_desc_t *current_iodesc=NULL;
+static iosystem_desc_t *pio_iosystem_list=NULL;
+static file_desc_t *pio_file_list = NULL;
+static file_desc_t *current_file=NULL;
+
+/** Add a new entry to the global list of open files.
+ *
+ * @param file pointer to the file_desc_t struct for the new file.
+ */
+void pio_add_to_file_list(file_desc_t *file)
+{
+    file_desc_t *cfile;
+
+    assert(file);
+
+    /* This file will be at the end of the list, and have no next. */
+    file->next = NULL;
+
+    /* Get a pointer to the global list of files. */
+    cfile = pio_file_list;
+
+    /* Keep a global pointer to the current file. */
+    current_file = file;
+
+    /* If there is nothing in the list, then file will be the first
+     * entry. Otherwise, move to end of the list. */
+    if (!cfile)
+        pio_file_list = file;
+    else
+    {
+        while (cfile->next)
+            cfile = cfile->next;
+        cfile->next = file;
+    }
+}
+
+/** Given ncid, find the file_desc_t data for an open file. The ncid
+ * used is the interally generated pio_ncid.
+ *
+ * @param ncid the PIO assigned ncid of the open file.
+ * @param cfile1 pointer to a pointer to a file_desc_t. The pointer
+ * will get a copy of the pointer to the file info.
+ *
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_get_file(int ncid, file_desc_t **cfile1)
+{
+    file_desc_t *cfile = NULL;
+
+    LOG((2, "pio_get_file ncid = %d", ncid));
+
+    /* Caller must provide this. */
+    if (!cfile1)
+        return PIO_EINVAL;
+
+    /* Find the file pointer. */
+    if (current_file && current_file->pio_ncid == ncid)
+        cfile = current_file;
+    else
+        for (cfile = pio_file_list; cfile; cfile = cfile->next)
+            if (cfile->pio_ncid == ncid)
+            {
+                current_file = cfile;
+                break;
+            }
+
+    /* If not found, return error. */
+    if (!cfile)
+        return PIO_EBADID;
+
+    /* We depend on every file having a pointer to the iosystem. */
+    if (!cfile->iosystem)
+        return PIO_EINVAL;
+
+    /* Let's just ensure we have a valid IO type. */
+    pioassert(iotype_is_valid(cfile->iotype), "invalid IO type", __FILE__, __LINE__);
+
+    /* Copy pointer to file info. */
+    *cfile1 = cfile;
+
+    return PIO_NOERR;
+}
+
+/** Delete a file from the list of open files.
+ *
+ * @param ncid ID of file to delete from list
+ * @returns 0 for success, error code otherwise
+ */
+int pio_delete_file_from_list(int ncid)
+{
+
+    file_desc_t *cfile, *pfile = NULL;
+
+    /* Look through list of open files. */
+    for (cfile = pio_file_list; cfile; cfile = cfile->next)
+    {
+        if (cfile->pio_ncid == ncid)
+        {
+            if (!pfile)
+                pio_file_list = cfile->next;
+            else
+                pfile->next = cfile->next;
+
+            if (current_file == cfile)
+                current_file = pfile;
+
+            /* Free the memory used for this file. */
+            free(cfile);
+            return PIO_NOERR;
+        }
+        pfile = cfile;
+    }
+
+    /* No file was found. */
+    return PIO_EBADID;
+}
+
+/** Delete iosystem info from list.
+ *
+ * @param piosysid the iosysid to delete
+ * @returns 0 on success, error code otherwise
+ */
+int pio_delete_iosystem_from_list(int piosysid)
+{
+    iosystem_desc_t *ciosystem, *piosystem = NULL;
+
+    LOG((1, "pio_delete_iosystem_from_list piosysid = %d", piosysid));
+
+    for (ciosystem = pio_iosystem_list; ciosystem; ciosystem = ciosystem->next)
+    {
+        if (ciosystem->iosysid == piosysid)
+        {
+            if (piosystem == NULL)
+                pio_iosystem_list = ciosystem->next;
+            else
+                piosystem->next = ciosystem->next;
+            free(ciosystem);
+            return PIO_NOERR;
+        }
+        piosystem = ciosystem;
+    }
+    return PIO_EBADID;
+}
+
+/** Add iosystem info to list.
+ *
+ * @param ios pointer to the iosystem_desc_t info to add.
+ * @returns 0 on success, error code otherwise
+ */
+int pio_add_to_iosystem_list(iosystem_desc_t *ios)
+{
+    iosystem_desc_t *cios;
+    int i = 1;
+
+    assert(ios);
+
+    ios->next = NULL;
+    cios = pio_iosystem_list;
+    if (!cios)
+        pio_iosystem_list = ios;
+    else
+    {
+        i++;
+        while (cios->next)
+        {
+            cios = cios->next;
+            i++;
+        }
+        cios->next = ios;
+    }
+
+    ios->iosysid = i << 16;
+
+    return ios->iosysid;
+}
+
+/** Get iosystem info from list.
+ *
+ * @param iosysid id of the iosystem
+ * @returns pointer to iosystem_desc_t, or NULL if not found.
+ */
+iosystem_desc_t *pio_get_iosystem_from_id(int iosysid)
+{
+    iosystem_desc_t *ciosystem;
+
+    LOG((2, "pio_get_iosystem_from_id iosysid = %d", iosysid));
+
+    for (ciosystem = pio_iosystem_list; ciosystem; ciosystem = ciosystem->next)
+        if (ciosystem->iosysid == iosysid)
+            return ciosystem;
+
+    return NULL;
+}
+
+/** Count the number of open iosystems.
+ *
+ * @param niosysid pointer that will get the number of open iosystems.
+ * @returns 0 for success.
+ */
+int
+pio_num_iosystem(int *niosysid)
+{
+    int count = 0;
+
+    /* Count the elements in the list. */
+    for (iosystem_desc_t *c = pio_iosystem_list; c; c = c->next)
+        count++;
+
+    /* Return count to caller via pointer. */
+    if (niosysid)
+        *niosysid = count;
+
+    return PIO_NOERR;
+}
+
+/** Add an iodesc.
+ *
+ * @param io_desc_t pointer to data to add to list.
+ * @returns the ioid of the newly added iodesc.
+ */
+int pio_add_to_iodesc_list(io_desc_t *iodesc)
+{
+    io_desc_t *ciodesc;
+    int imax = 512;
+
+    iodesc->next = NULL;
+    if (pio_iodesc_list == NULL)
+        pio_iodesc_list = iodesc;
+    else{
+        imax++;
+        for (ciodesc = pio_iodesc_list; ciodesc->next; ciodesc=ciodesc->next, imax=ciodesc->ioid + 1);
+        ciodesc->next = iodesc;
+    }
+    iodesc->ioid = imax;
+    current_iodesc = iodesc;
+
+    return iodesc->ioid;
+}
+
+/** Get an iodesc.
+ *
+ * @param ioid ID of iodesc to get.
+ * @returns pointer to the iodesc struc.
+ */
+io_desc_t *pio_get_iodesc_from_id(int ioid)
+{
+    io_desc_t *ciodesc = NULL;
+
+    if (current_iodesc != NULL && current_iodesc->ioid == abs(ioid))
+        ciodesc = current_iodesc;
+    else
+        for (ciodesc = pio_iodesc_list; ciodesc; ciodesc = ciodesc->next)
+            if (ciodesc->ioid == abs(ioid))
+            {
+                current_iodesc = ciodesc;
+                break;
+            }
+
+    return ciodesc;
+}
+
+/** Delete an iodesc.
+ *
+ * @param ioid ID of iodesc to delete.
+ * @returns 0 on success, error code otherwise.
+ */
+int pio_delete_iodesc_from_list(int ioid)
+{
+    io_desc_t *ciodesc, *piodesc = NULL;
+
+    for (ciodesc = pio_iodesc_list; ciodesc; ciodesc = ciodesc->next)
+    {
+        if (ciodesc->ioid == ioid)
+        {
+            if (piodesc == NULL)
+                pio_iodesc_list = ciodesc->next;
+            else
+                piodesc->next = ciodesc->next;
+
+            if (current_iodesc == ciodesc)
+                current_iodesc = pio_iodesc_list;
+            free(ciodesc);
+            return PIO_NOERR;
+        }
+        piodesc = ciodesc;
+    }
+    return PIO_EBADID;
+}
diff --git a/src/externals/pio2/src/clib/pio_msg.c b/src/externals/pio2/src/clib/pio_msg.c
new file mode 100644
index 00000000000..75d2a8bfea4
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_msg.c
@@ -0,0 +1,2511 @@
+/** @file
+ *
+ * PIO async message handling. This file contains the code which
+ * runs on the IO nodes when async is in use. This code waits for
+ * messages from the computation nodes, and responds to messages by
+ * running the appropriate netCDF function.
+ *
+ * @author Ed Hartnett
+ */
+
+#include 
+#include 
+#include 
+
+#ifdef PIO_ENABLE_LOGGING
+extern int my_rank;
+extern int pio_log_level;
+#endif /* PIO_ENABLE_LOGGING */
+
+/** This function is run on the IO tasks to handle nc_inq_type*()
+ * functions.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_type_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int xtype;
+    char name_present, size_present;
+    char *namep = NULL, name[NC_MAX_NAME + 1];
+    PIO_Offset *sizep = NULL, size;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_type_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Handle null pointer issues. */
+    if (name_present)
+        namep = name;
+    if (size_present)
+        sizep = &size;
+
+    /* Call the function. */
+    if ((ret = PIOc_inq_type(ncid, xtype, namep, sizep)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "inq_type_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to find netCDF file
+ * format.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_format_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int *formatp = NULL, format;
+    char format_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_format_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_format_handler got parameters ncid = %d format_present = %d",
+         ncid, format_present));
+
+    /* Manage NULL pointers. */
+    if (format_present)
+        formatp = &format;
+
+    /* Call the function. */
+    if ((ret = PIOc_inq_format(ncid, formatp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    if (formatp)
+        LOG((2, "inq_format_handler format = %d", *formatp));
+    LOG((1, "inq_format_handler succeeded!"));
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to set the file fill mode.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, error code otherwise.
+ * @internal
+ */
+int set_fill_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int fillmode;
+    int old_modep_present;
+    int old_mode, *old_modep = NULL;
+    int mpierr;
+    int ret;
+
+    LOG((1, "set_fill_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "set_fill_handler got parameters ncid = %d fillmode = %d old_modep_present = %d",
+         ncid, fillmode, old_modep_present));
+
+    /* Manage NULL pointers. */
+    if (old_modep_present)
+        old_modep = &old_mode;
+
+    /* Call the function. */
+    if ((ret = PIOc_set_fill(ncid, fillmode, old_modep)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "set_fill_handler succeeded!"));
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to create a netCDF file.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int create_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int len;
+    int iotype;
+    int mode;
+    int mpierr;
+    int ret;
+
+    LOG((1, "create_file_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Get space for the filename. */
+    char filename[len + 1];
+
+    if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "create_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
+         len, filename, iotype, mode));
+
+    /* Call the create file function. */
+    if ((ret = PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "create_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to close a netCDF file. It is
+ * only ever run on the IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int close_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int mpierr;
+    int ret;
+
+    LOG((1, "close_file_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "create_file_handler got parameter ncid = %d", ncid));
+
+    /* Call the close file function. */
+    if ((ret = PIOc_closefile(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "close_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to inq a netCDF file. It is
+ * only ever run on the IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int ndims, nvars, ngatts, unlimdimid;
+    int *ndimsp = NULL, *nvarsp = NULL, *ngattsp = NULL, *unlimdimidp = NULL;
+    char ndims_present, nvars_present, ngatts_present, unlimdimid_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "inq_handler ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d",
+         ndims_present, nvars_present, ngatts_present, unlimdimid_present));
+
+    /* NULLs passed in to any of the pointers in the original call
+     * need to be matched with NULLs here. Assign pointers where
+     * non-NULL pointers were passed in. */
+    if (ndims_present)
+        ndimsp = &ndims;
+    if (nvars_present)
+        nvarsp = &nvars;
+    if (ngatts_present)
+        ngattsp = &ngatts;
+    if (unlimdimid_present)
+        unlimdimidp = &unlimdimid;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Do an inq_dim on a netCDF dimension. This function is only run on
+ * IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_dim_handler(iosystem_desc_t *ios, int msg)
+{
+    int ncid;
+    int dimid;
+    char name_present, len_present;
+    char *dimnamep = NULL;
+    PIO_Offset *dimlenp = NULL;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_dim_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_handler name_present = %d len_present = %d", name_present,
+         len_present));
+
+    /* Set the non-null pointers. */
+    if (name_present)
+        dimnamep = dimname;
+    if (len_present)
+        dimlenp = &dimlen;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_dim(ncid, dimid, dimnamep, dimlenp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Do an inq_dimid on a netCDF dimension name. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_dimid_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int *dimidp = NULL, dimid;
+    int mpierr;
+    int id_present;
+    int ret;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+
+    LOG((1, "inq_dimid_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d",
+         ncid, namelen, name, id_present));
+
+    /* Set non-null pointer. */
+    if (id_present)
+        dimidp = &dimid;
+
+    /* Call the inq_dimid function. */
+    if ((ret = PIOc_inq_dimid(ncid, name, dimidp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute inquiry operations. This code only runs on IO
+ * tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_att_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    int ret;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    nc_type xtype, *xtypep = NULL;
+    PIO_Offset len, *lenp = NULL;
+    char xtype_present, len_present;
+
+    LOG((1, "inq_att_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster,
+                            ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Match NULLs in collective function call. */
+    if (xtype_present)
+        xtypep = &xtype;
+    if (len_present)
+        lenp = &len;
+
+    /* Call the function to learn about the attribute. */
+    if ((ret = PIOc_inq_att(ncid, varid, name, xtypep, lenp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute inquiry operations. This code only runs on IO
+ * tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_attname_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int attnum;
+    char name[NC_MAX_NAME + 1], *namep = NULL;
+    char name_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_att_name_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&attnum, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_attname_handler got ncid = %d varid = %d attnum = %d name_present = %d",
+         ncid, varid, attnum, name_present));
+
+    /* Match NULLs in collective function call. */
+    if (name_present)
+        namep = name;
+
+    /* Call the function to learn about the attribute. */
+    if ((ret = PIOc_inq_attname(ncid, varid, attnum, namep)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute inquiry operations. This code only runs on IO
+ * tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_attid_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    int id, *idp = NULL;
+    char id_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_attid_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_attid_handler got ncid = %d varid = %d id_present = %d",
+         ncid, varid, id_present));
+
+    /* Match NULLs in collective function call. */
+    if (id_present)
+        idp = &id;
+
+    /* Call the function to learn about the attribute. */
+    if ((ret = PIOc_inq_attid(ncid, varid, name, idp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int att_put_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    int ret;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    PIO_Offset attlen;  /* Number of elements in att array. */
+    nc_type atttype;    /* Type of att in file. */
+    PIO_Offset atttype_len; /* Length in bytes of one elementy of type atttype. */
+    nc_type memtype;    /* Type of att data in memory. */
+    PIO_Offset memtype_len; /* Length of element of memtype. */
+    void *op;
+
+    LOG((1, "att_put_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Allocate memory for the attribute data. */
+    if (!(op = malloc(attlen * memtype_len)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(op, attlen * memtype_len, MPI_BYTE, 0, ios->intercomm)))
+    {
+        free(op);
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    LOG((1, "att_put_handler ncid = %d varid = %d namelen = %d name = %s"
+         "atttype = %d attlen = %d atttype_len = %d memtype = %d memtype_len = 5d",
+         ncid, varid, namelen, name, atttype, attlen, atttype_len, memtype, memtype_len));
+
+    /* Call the function to write the attribute. */
+    ret = PIOc_put_att_tc(ncid, varid, name, atttype, attlen, memtype, op);
+
+    /* Free resources. */
+    free(op);
+
+    /* Did it work? */
+    if (ret)
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((2, "att_put_handler complete!"));
+    return PIO_NOERR;
+}
+
+/** Handle attribute operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int att_get_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    PIO_Offset attlen;
+    nc_type atttype;        /* Type of att in file. */
+    PIO_Offset atttype_len; /* Length in bytes of an element of attype. */
+    nc_type memtype;        /* Type of att in memory. */
+    PIO_Offset memtype_len; /* Length in bytes of an element of memype. */
+    int *ip;
+    int iotype;
+    int ret;
+
+    LOG((1, "att_get_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d"
+         " atttype = %d attlen = %d atttype_len = %d memtype = %d memtype_len = %d",
+         ncid, varid, namelen, name, iotype, atttype, attlen, atttype_len, memtype, memtype_len));
+
+    /* Allocate space for the attribute data. */
+    if (!(ip = malloc(attlen * memtype_len)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Call the function to read the attribute. */
+    ret = PIOc_get_att_tc(ncid, varid, name, memtype, ip);
+
+    /* Free resources. */
+    free(ip);
+
+    /* Did it work? */
+    if (ret)
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle var put operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int put_vars_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    PIO_Offset typelen;  /** Length (in bytes) of this type. */
+    nc_type xtype;       /** Type of the data being written. */
+    char stride_present; /** Zero if user passed a NULL stride. */
+    PIO_Offset *startp = NULL;
+    PIO_Offset *countp = NULL;
+    PIO_Offset *stridep = NULL;
+    int ndims;           /** Number of dimensions. */
+    void *buf;           /** Buffer for data storage. */
+    PIO_Offset num_elem; /** Number of data elements in the buffer. */
+    int mpierr;          /** Error code from MPI function calls. */
+
+    LOG((1, "put_vars_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Now we know how big to make these arrays. */
+    PIO_Offset start[ndims], count[ndims], stride[ndims];
+
+    if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims));
+    if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (stride_present)
+        if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "put_vars_handler ncid = %d varid = %d ndims = %d "
+         "stride_present = %d xtype = %d num_elem = %d typelen = %d",
+         ncid, varid, ndims, stride_present, xtype, num_elem, typelen));
+
+    for (int d = 0; d < ndims; d++)
+    {
+        LOG((2, "start[%d] = %d", d, start[d]));
+        LOG((2, "count[%d] = %d", d, count[d]));
+        if (stride_present)
+            LOG((2, "stride[%d] = %d", d, stride[d]));
+    }
+
+    /* Allocate room for our data. */
+    if (!(buf = malloc(num_elem * typelen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Get the data. */
+    if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm)))
+    {
+        free(buf);
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* for (int e = 0; e < num_elem; e++) */
+    /*  LOG((2, "element %d = %d", e, ((int *)buf)[e])); */
+
+    /* Set the non-NULL pointers. */
+    startp = start;
+    countp = count;
+    if (stride_present)
+        stridep = stride;
+
+    /* Call the function to write the data. No need to check return
+     * values, they are bcast to computation tasks inside function. */
+    switch(xtype)
+    {
+    case NC_BYTE:
+        PIOc_put_vars_schar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_CHAR:
+        PIOc_put_vars_text(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_SHORT:
+        PIOc_put_vars_short(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT:
+        PIOc_put_vars_int(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case PIO_LONG_INTERNAL:
+        PIOc_put_vars_long(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_FLOAT:
+        PIOc_put_vars_float(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_DOUBLE:
+        PIOc_put_vars_double(ncid, varid, startp, countp, stridep, buf);
+        break;
+#ifdef _NETCDF4
+    case NC_UBYTE:
+        PIOc_put_vars_uchar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_USHORT:
+        PIOc_put_vars_ushort(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT:
+        PIOc_put_vars_uint(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT64:
+        PIOc_put_vars_longlong(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT64:
+        PIOc_put_vars_ulonglong(ncid, varid, startp, countp, stridep, buf);
+        break;
+        /* case NC_STRING: */
+        /*      PIOc_put_vars_string(ncid, varid, startp, countp, */
+        /*                                stridep, (void *)buf); */
+        /*      break; */
+        /*    default:*/
+        /* PIOc_put_vars(ncid, varid, startp, countp, */
+        /*                   stridep, buf); */
+#endif /* _NETCDF4 */
+    }
+
+    free(buf);
+
+    return PIO_NOERR;
+}
+
+/** Handle var get operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int get_vars_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    PIO_Offset typelen; /** Length (in bytes) of this type. */
+    nc_type xtype; /** Type of the data being written. */
+    char stride_present;
+    PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL;
+    int ndims; /** Number of dimensions. */
+    void *buf; /** Buffer for data storage. */
+    PIO_Offset num_elem; /** Number of data elements in the buffer. */
+
+    LOG((1, "get_vars_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Now we know how big to make these arrays. */
+    PIO_Offset start[ndims], count[ndims], stride[ndims];
+
+    if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims));
+    if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (stride_present)
+        if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "get_vars_handler ncid = %d varid = %d ndims = %d "
+         "stride_present = %d xtype = %d num_elem = %d typelen = %d",
+         ncid, varid, ndims, stride_present, xtype, num_elem, typelen));
+
+    for (int d = 0; d < ndims; d++)
+    {
+        LOG((2, "start[%d] = %d", d, start[d]));
+        LOG((2, "count[%d] = %d", d, count[d]));
+        if (stride_present)
+            LOG((2, "stride[%d] = %d", d, stride[d]));
+    }
+
+    /* Allocate room for our data. */
+    if (!(buf = malloc(num_elem * typelen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Set the non-NULL pointers. */
+    startp = start;
+    countp = count;
+    if (stride_present)
+        stridep = stride;
+
+    /* Call the function to read the data. */
+    switch(xtype)
+    {
+    case NC_BYTE:
+        PIOc_get_vars_schar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_CHAR:
+        PIOc_get_vars_text(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_SHORT:
+        PIOc_get_vars_short(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT:
+        PIOc_get_vars_int(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case PIO_LONG_INTERNAL:
+        PIOc_get_vars_long(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_FLOAT:
+        PIOc_get_vars_float(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_DOUBLE:
+        PIOc_get_vars_double(ncid, varid, startp, countp, stridep, buf);
+        break;
+#ifdef _NETCDF4
+    case NC_UBYTE:
+        PIOc_get_vars_uchar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_USHORT:
+        PIOc_get_vars_ushort(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT:
+        PIOc_get_vars_uint(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT64:
+        PIOc_get_vars_longlong(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT64:
+        PIOc_get_vars_ulonglong(ncid, varid, startp, countp, stridep, buf);
+        break;
+        /* case NC_STRING: */
+        /*      PIOc_get_vars_string(ncid, varid, startp, countp, */
+        /*                                stridep, (void *)buf); */
+        /*      break; */
+        /*    default:*/
+        /* PIOc_get_vars(ncid, varid, startp, countp, */
+        /*                   stridep, buf); */
+#endif /* _NETCDF4 */
+    }
+
+    free(buf);
+    LOG((1, "get_vars_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var on a netCDF variable. This function is only run on
+ * IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    char name_present, xtype_present, ndims_present, dimids_present, natts_present;
+    char name[NC_MAX_NAME + 1], *namep = NULL;
+    nc_type xtype, *xtypep = NULL;
+    int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL;
+    int ndims, dimids[NC_MAX_DIMS], natts;
+    int ret;
+
+    LOG((1, "inq_var_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_handler ncid = %d varid = %d name_present = %d xtype_present = %d ndims_present = %d "
+         "dimids_present = %d natts_present = %d",
+         ncid, varid, name_present, xtype_present, ndims_present, dimids_present, natts_present));
+
+    /* Set the non-NULL pointers. */
+    if (name_present)
+        namep = name;
+    if (xtype_present)
+        xtypep = &xtype;
+    if (ndims_present)
+        ndimsp = &ndims;
+    if (dimids_present)
+        dimidsp = dimids;
+    if (natts_present)
+        nattsp = &natts;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    if (ndims_present)
+        LOG((2, "inq_var_handler ndims = %d", ndims));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_chunking on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_chunking_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char storage_present, chunksizes_present;
+    int storage, *storagep = NULL;
+    PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "inq_var_chunking_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_handler ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
+         ncid, varid, storage_present, chunksizes_present));
+
+    /* Set the non-NULL pointers. */
+    if (storage_present)
+        storagep = &storage;
+    if (chunksizes_present)
+        chunksizesp = chunksizes;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var_chunking(ncid, varid, storagep, chunksizesp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_fill on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_fill_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char fill_mode_present, fill_value_present;
+    PIO_Offset type_size;
+    int fill_mode, *fill_modep = NULL;
+    PIO_Offset *fill_value, *fill_valuep = NULL;
+    int mpierr;
+
+    assert(ios);
+    LOG((1, "inq_var_fill_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_mode_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_fill_handler ncid = %d varid = %d type_size = %lld, fill_mode_present = %d fill_value_present = %d",
+         ncid, varid, type_size, fill_mode_present, fill_value_present));
+
+    /* If we need to, alocate storage for fill value. */
+    if (fill_value_present)
+        if (!(fill_value = malloc(type_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Set the non-NULL pointers. */
+    if (fill_mode_present)
+        fill_modep = &fill_mode;
+    if (fill_value_present)
+        fill_valuep = fill_value;
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_fill(ncid, varid, fill_modep, fill_valuep);
+
+    /* Free fill value storage if we allocated some. */
+    if (fill_value_present)
+        free(fill_value);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_endian on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_endian_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char endian_present;
+    int endian, *endianp = NULL;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "inq_var_endian_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_endian_handler ncid = %d varid = %d endian_present = %d", ncid, varid,
+         endian_present));
+
+    /* Set the non-NULL pointers. */
+    if (endian_present)
+        endianp = &endian;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var_endian(ncid, varid, endianp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_deflate on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_deflate_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char shuffle_present;
+    char deflate_present;
+    char deflate_level_present;
+    int shuffle, *shufflep;
+    int deflate, *deflatep;
+    int deflate_level, *deflate_levelp;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "inq_var_deflate_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (shuffle_present && !mpierr)
+        if ((mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (deflate_present && !mpierr)
+        if ((mpierr = MPI_Bcast(&deflate, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (deflate_level_present && !mpierr)
+        if ((mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_var_handler ncid = %d varid = %d shuffle_present = %d deflate_present = %d "
+         "deflate_level_present = %d", ncid, varid, shuffle_present, deflate_present,
+         deflate_level_present));
+
+    /* Set the non-NULL pointers. */
+    if (shuffle_present)
+        shufflep = &shuffle;
+    if (deflate_present)
+        deflatep = &deflate;
+    if (deflate_level_present)
+        deflate_levelp = &deflate_level;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var_deflate(ncid, varid, shufflep, deflatep, deflate_levelp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Do an inq_varid on a netCDF variable name. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_varid_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    int ret;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Call the inq_dimid function. */
+    if ((ret = PIOc_inq_varid(ncid, name, &varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to sync a netCDF file.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int sync_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int mpierr;
+    int ret;
+
+    LOG((1, "sync_file_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "sync_file_handler got parameter ncid = %d", ncid));
+
+    /* Call the sync file function. */
+    if ((ret = PIOc_sync(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((2, "sync_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to enddef a netCDF file.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int change_def_file_handler(iosystem_desc_t *ios, int msg)
+{
+    int ncid;
+    int mpierr;
+
+    LOG((1, "change_def_file_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Call the function. */
+    if (msg == PIO_MSG_ENDDEF)
+        PIOc_enddef(ncid);
+    else
+        PIOc_redef(ncid);
+
+    LOG((1, "change_def_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to define a netCDF
+ *  variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int def_var_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int varid;
+    nc_type xtype;
+    int ndims;
+    int *dimids;
+
+    LOG((1, "def_var_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (!(dimids = malloc(ndims * sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(dimids, ndims, MPI_INT, 0, ios->intercomm)))
+    {
+        free(dimids);
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    LOG((1, "def_var_handler got parameters namelen = %d "
+         "name = %s ncid = %d", namelen, name, ncid));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var(ncid, name, xtype, ndims, dimids, &varid)))
+    {
+        free(dimids);
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    }
+
+    /* Free resources. */
+    free(dimids);
+
+    LOG((1, "def_var_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define chunking for a
+ *  netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_chunking_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int ndims;
+    int storage;
+    char chunksizes_present;
+    PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "def_var_chunking_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&storage, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (chunksizes_present)
+        if ((mpierr = MPI_Bcast(chunksizes, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "def_var_chunking_handler got parameters ncid = %d varid = %d storage = %d "
+         "ndims = %d chunksizes_present = %d", ncid, varid, storage, ndims, chunksizes_present));
+
+    /* Set the non-NULL pointers. */
+    if (chunksizes_present)
+        chunksizesp = chunksizes;
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var_chunking(ncid, varid, storage, chunksizesp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_chunking_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define fill mode and fill
+ * value.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_fill_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int fill_mode;
+    char fill_value_present;
+    PIO_Offset type_size;
+    PIO_Offset *fill_valuep = NULL;
+    int mpierr;
+
+    assert(ios);
+    LOG((1, "def_var_fill_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (fill_value_present)
+    {
+        if (!(fill_valuep = malloc(type_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, 0, ios->intercomm)))
+        {
+            free(fill_valuep);
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        }
+    }
+    LOG((1, "def_var_fill_handler got parameters ncid = %d varid = %d fill_mode = %d "
+         "type_size = %lld fill_value_present = %d", ncid, varid, fill_mode, type_size, fill_value_present));
+
+    /* Call the function. */
+    PIOc_def_var_fill(ncid, varid, fill_mode, fill_valuep);
+
+    /* Free memory allocated for the fill value. */
+    if (fill_valuep)
+        free(fill_valuep);
+
+    LOG((1, "def_var_fill_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define endianness for a
+ * netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_endian_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int endian;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "def_var_endian_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&endian, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "def_var_endian_handler got parameters ncid = %d varid = %d endain = %d ",
+         ncid, varid, endian));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var_endian(ncid, varid, endian)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_chunking_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define deflate settings for
+ * a netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_deflate_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int shuffle;
+    int deflate;
+    int deflate_level;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "def_var_deflate_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "def_var_deflate_handler got parameters ncid = %d varid = %d shuffle = %d ",
+         "deflate = %d deflate_level = %d", ncid, varid, shuffle, deflate, deflate_level));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var_deflate(ncid, varid, shuffle, deflate, deflate_level)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_deflate_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define chunk cache settings
+ * for a netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int set_var_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    PIO_Offset size;
+    PIO_Offset nelems;
+    float preemption;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    assert(ios);
+    LOG((1, "set_var_chunk_cache_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "set_var_chunk_cache_handler got params ncid = %d varid = %d size = %d "
+         "nelems = %d preemption = %g", ncid, varid, size, nelems, preemption));
+
+    /* Call the function. */
+    if ((ret = PIOc_set_var_chunk_cache(ncid, varid, size, nelems, preemption)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to define a netCDF
+ * dimension.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int def_dim_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int len, namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int dimid;
+
+    LOG((1, "def_dim_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "def_dim_handler got parameters namelen = %d "
+         "name = %s len = %d ncid = %d", namelen, name, len, ncid));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_dim(ncid, name, len, &dimid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_dim_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to rename a netCDF
+ * dimension.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int rename_dim_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int dimid;
+
+    LOG((1, "rename_dim_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "rename_dim_handler got parameters namelen = %d "
+         "name = %s ncid = %d dimid = %d", namelen, name, ncid, dimid));
+
+    /* Call the function. */
+    if ((ret = PIOc_rename_dim(ncid, dimid, name)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "rename_dim_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to rename a netCDF
+ * dimension.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int rename_var_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int varid;
+
+    LOG((1, "rename_var_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "rename_var_handler got parameters namelen = %d "
+         "name = %s ncid = %d varid = %d", namelen, name, ncid, varid));
+
+    /* Call the function. */
+    if ((ret = PIOc_rename_var(ncid, varid, name)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "rename_var_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to rename a netCDF
+ * attribute.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int rename_att_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int namelen, newnamelen;
+    char name[PIO_MAX_NAME + 1], newname[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+
+    LOG((1, "rename_att_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(newname, newnamelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "rename_att_handler got parameters namelen = %d name = %s ncid = %d varid = %d "
+         "newnamelen = %d newname = %s", namelen, name, ncid, varid, newnamelen, newname));
+
+    /* Call the function. */
+    if ((ret = PIOc_rename_att(ncid, varid, name, newname)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "rename_att_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to delete a netCDF
+ * attribute.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int delete_att_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+
+    LOG((1, "delete_att_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "delete_att_handler namelen = %d name = %s ncid = %d varid = %d ",
+         namelen, name, ncid, varid));
+
+    /* Call the function. */
+    if ((ret = PIOc_del_att(ncid, varid, name)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "delete_att_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to open a netCDF file.
+ *
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int open_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int len;
+    int iotype;
+    int mode;
+    int mpierr;
+
+    LOG((1, "open_file_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "open_file_handler got parameter len = %d", len));
+
+    /* Get space for the filename. */
+    char filename[len + 1];
+
+    if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
+         len, filename, iotype, mode));
+
+    /* Call the open file function. Errors are handling within
+     * function, so return code can be ignored. */
+    PIOc_openfile(ios->iosysid, &ncid, &iotype, filename, mode);
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to delete a netCDF file.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int delete_file_handler(iosystem_desc_t *ios)
+{
+    int len;
+    int mpierr;
+    int ret;
+
+    LOG((1, "delete_file_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Get space for the filename. */
+    char filename[len + 1];
+
+    if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "delete_file_handler got parameters len = %d filename = %s",
+         len, filename));
+
+    /* Call the delete file function. */
+    if ((ret = PIOc_deletefile(ios->iosysid, filename)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "delete_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** 
+ * This function is run on the IO tasks to initialize a decomposition.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int initdecomp_dof_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int basetype;
+    int ndims;
+    int maplen;
+    int ioid;
+    char rearranger_present;
+    int rearranger;
+    int *rearrangerp = NULL;
+    char iostart_present;
+    PIO_Offset *iostartp = NULL;
+    char iocount_present;
+    PIO_Offset *iocountp = NULL;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "initdecomp_dof_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&basetype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Now we know the size of these arrays. */
+    int dims[ndims];
+    PIO_Offset iostart[ndims];
+    PIO_Offset iocount[ndims];
+
+    if ((mpierr = MPI_Bcast(dims, ndims, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&maplen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PIO_Offset compmap[maplen];
+
+    if ((mpierr = MPI_Bcast(compmap, maplen, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (rearranger_present)
+        if ((mpierr = MPI_Bcast(&rearranger, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (iostart_present)
+        if ((mpierr = MPI_Bcast(iostart, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (iocount_present)
+        if ((mpierr = MPI_Bcast(iocount, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "initdecomp_dof_handler iosysid = %d basetype = %d ndims = %d maplen = %d "
+         "rearranger_present = %d iostart_present = %d iocount_present = %d ",
+         iosysid, basetype, ndims, maplen, rearranger_present, iostart_present, iocount_present));
+
+    if (rearranger_present)
+        rearrangerp = &rearranger;
+    if (iostart_present)
+        iostartp = iostart;
+    if (iocount_present)
+        iocountp = iocount;
+
+    /* Call the function. */
+    ret = PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap, &ioid, rearrangerp,
+                          iostartp, iocountp);
+    
+    LOG((1, "PIOc_InitDecomp returned %d", ret));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to...
+ * NOTE: not yet implemented
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int writedarray_handler(iosystem_desc_t *ios)
+{
+    assert(ios);
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to...
+ * NOTE: not yet implemented
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int readdarray_handler(iosystem_desc_t *ios)
+{
+    assert(ios);
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to set the error handler.
+ * NOTE: not yet implemented
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int seterrorhandling_handler(iosystem_desc_t *ios)
+{
+    int method;
+    int old_method_present;
+    int old_method;
+    int *old_methodp = NULL;
+    int mpierr;
+    int ret;
+
+    LOG((1, "seterrorhandling_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&method, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((1, "seterrorhandling_handler got parameters method = %d old_method_present = %d",
+         method, old_method_present));
+
+    if (old_method_present)
+        old_methodp = &old_method;
+
+    /* Call the function. */
+    if ((ret = PIOc_set_iosystem_error_handling(ios->iosysid, method, old_methodp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "seterrorhandling_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to set the chunk cache
+ * parameters for netCDF-4.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int set_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int iotype;
+    PIO_Offset size;
+    PIO_Offset nelems;
+    float preemption;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "set_chunk_cache_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "set_chunk_cache_handler got params iosysid = %d iotype = %d size = %d "
+         "nelems = %d preemption = %g", iosysid, iotype, size, nelems, preemption));
+
+    /* Call the function. */
+    if ((ret = PIOc_set_chunk_cache(iosysid, iotype, size, nelems, preemption)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "set_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to get the chunk cache
+ * parameters for netCDF-4.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int get_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int iotype;
+    char size_present, nelems_present, preemption_present;
+    PIO_Offset size, *sizep;
+    PIO_Offset nelems, *nelemsp;
+    float preemption, *preemptionp;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "get_chunk_cache_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "get_chunk_cache_handler got params iosysid = %d iotype = %d size_present = %d "
+         "nelems_present = %d preemption_present = %g", iosysid, iotype, size_present,
+         nelems_present, preemption_present));
+
+    /* Set the non-NULL pointers. */
+    if (size_present)
+        sizep = &size;
+    if (nelems_present)
+        nelemsp = &nelems;
+    if (preemption_present)
+        preemptionp = &preemption;
+
+    /* Call the function. */
+    if ((ret = PIOc_get_chunk_cache(iosysid, iotype, sizep, nelemsp, preemptionp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "get_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to get the variable chunk
+ * cache parameters for netCDF-4.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int get_var_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char size_present, nelems_present, preemption_present;
+    PIO_Offset size, *sizep;
+    PIO_Offset nelems, *nelemsp;
+    float preemption, *preemptionp;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "get_var_chunk_cache_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "get_var_chunk_cache_handler got params ncid = %d varid = %d size_present = %d "
+         "nelems_present = %d preemption_present = %g", ncid, varid, size_present,
+         nelems_present, preemption_present));
+
+    /* Set the non-NULL pointers. */
+    if (size_present)
+        sizep = &size;
+    if (nelems_present)
+        nelemsp = &nelems;
+    if (preemption_present)
+        preemptionp = &preemption;
+
+    /* Call the function. */
+    if ((ret = PIOc_get_var_chunk_cache(ncid, varid, sizep, nelemsp, preemptionp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "get_var_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to free the decomp hanlder.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int freedecomp_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int ioid;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "freedecomp_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ioid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "freedecomp_handler iosysid = %d ioid = %d", iosysid, ioid));
+
+    /* Call the function. */
+    ret = PIOc_freedecomp(iosysid, ioid);
+    
+    LOG((1, "PIOc_freedecomp returned %d", ret));
+    return PIO_NOERR;
+}
+
+/** Handle the finalize call.
+ *
+ * @param ios pointer to the iosystem info
+ * @param index
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int finalize_handler(iosystem_desc_t *ios, int index)
+{
+    int iosysid;
+    int mpierr;
+    int ret;
+
+    LOG((1, "finalize_handler called index = %d", index));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "finalize_handler got parameter iosysid = %d", iosysid));
+
+    /* Call the function. */
+    LOG((2, "finalize_handler calling PIOc_finalize for iosysid = %d",
+         iosysid));
+    if ((ret = PIOc_finalize(iosysid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "finalize_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is called by the IO tasks.  This function will not
+ * return, unless there is an error.
+ *
+ * @param io_rank
+ * @param component_count number of computation components
+ * @param iosys pointer to pointer to iosystem info
+ * @param io_comm MPI communicator for IO
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
+                     MPI_Comm io_comm)
+{
+    iosystem_desc_t *my_iosys;
+    int msg = 0;
+    MPI_Request req[component_count];
+    MPI_Status status;
+    int index;
+    int mpierr;
+    int ret = PIO_NOERR;
+    int open_components = component_count;
+
+    LOG((1, "pio_msg_handler2 called"));
+    assert(iosys);
+
+    /* Have IO comm rank 0 (the ioroot) register to receive
+     * (non-blocking) for a message from each of the comproots. */
+    if (!io_rank)
+    {
+        for (int cmp = 0; cmp < component_count; cmp++)
+        {
+            my_iosys = iosys[cmp];
+            LOG((1, "about to call MPI_Irecv union_comm = %d", my_iosys->union_comm));
+            if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG,
+                                    my_iosys->union_comm, &req[cmp])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((1, "MPI_Irecv req[%d] = %d", cmp, req[cmp]));
+        }
+    }
+
+    /* If the message is not -1, keep processing messages. */
+    while (msg != -1)
+    {
+        LOG((3, "pio_msg_handler2 at top of loop"));
+
+        /* Wait until any one of the requests are complete. Once it
+         * returns, the Waitany function automatically sets the
+         * appropriate member of the req array to MPI_REQUEST_NULL. */
+        if (!io_rank)
+        {
+            LOG((1, "about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d",
+                 req[0], MPI_REQUEST_NULL));
+            for (int c = 0; c < component_count; c++)
+                LOG((2, "req[%d] = %d", c, req[c]));
+            if ((mpierr = MPI_Waitany(component_count, req, &index, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "Waitany returned index = %d req[%d] = %d", index, index, req[index]));
+        }
+
+        /* Broadcast the index of the computational component that
+         * originated the request to the rest of the IO tasks. */
+        LOG((3, "About to do Bcast of index = %d io_comm = %d", index, io_comm));
+        if ((mpierr = MPI_Bcast(&index, 1, MPI_INT, 0, io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "index MPI_Bcast complete index = %d", index));
+
+        /* Set the correct iosys depending on the index. */
+        my_iosys = iosys[index];
+
+        /* Broadcast the msg value to the rest of the IO tasks. */
+        LOG((3, "about to call msg MPI_Bcast my_iosys->io_comm = %d", my_iosys->io_comm));
+        if ((mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, my_iosys->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((1, "pio_msg_handler2 msg MPI_Bcast complete msg = %d", msg));
+
+        /* Handle the message. This code is run on all IO tasks. */
+        switch (msg)
+        {
+        case PIO_MSG_INQ_TYPE:
+            inq_type_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_FORMAT:
+            inq_format_handler(my_iosys);
+            break;
+        case PIO_MSG_CREATE_FILE:
+            create_file_handler(my_iosys);
+            LOG((2, "returned from create_file_handler"));
+            break;
+        case PIO_MSG_SYNC:
+            sync_file_handler(my_iosys);
+            break;
+        case PIO_MSG_ENDDEF:
+        case PIO_MSG_REDEF:
+            LOG((2, "calling change_def_file_handler"));
+            change_def_file_handler(my_iosys, msg);
+            LOG((2, "returned from change_def_file_handler"));
+            break;
+        case PIO_MSG_OPEN_FILE:
+            open_file_handler(my_iosys);
+            break;
+        case PIO_MSG_CLOSE_FILE:
+            close_file_handler(my_iosys);
+            break;
+        case PIO_MSG_DELETE_FILE:
+            delete_file_handler(my_iosys);
+            break;
+        case PIO_MSG_RENAME_DIM:
+            rename_dim_handler(my_iosys);
+            break;
+        case PIO_MSG_RENAME_VAR:
+            rename_var_handler(my_iosys);
+            break;
+        case PIO_MSG_RENAME_ATT:
+            rename_att_handler(my_iosys);
+            break;
+        case PIO_MSG_DEL_ATT:
+            delete_att_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_DIM:
+            def_dim_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR:
+            def_var_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_CHUNKING:
+            def_var_chunking_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_FILL:
+            def_var_fill_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_ENDIAN:
+            def_var_endian_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_DEFLATE:
+            def_var_deflate_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_ENDIAN:
+            inq_var_endian_handler(my_iosys);
+            break;
+        case PIO_MSG_SET_VAR_CHUNK_CACHE:
+            set_var_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_VAR_CHUNK_CACHE:
+            get_var_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ:
+            inq_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_DIM:
+            inq_dim_handler(my_iosys, msg);
+            break;
+        case PIO_MSG_INQ_DIMID:
+            inq_dimid_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR:
+            inq_var_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_CHUNKING:
+            inq_var_chunking_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_FILL:
+            inq_var_fill_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_DEFLATE:
+            inq_var_deflate_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_ATT:
+            ret = att_get_handler(my_iosys);
+            break;
+        case PIO_MSG_PUT_ATT:
+            ret = att_put_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VARID:
+            inq_varid_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_ATT:
+            inq_att_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_ATTNAME:
+            inq_attname_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_ATTID:
+            inq_attid_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_VARS:
+            get_vars_handler(my_iosys);
+            break;
+        case PIO_MSG_PUT_VARS:
+            put_vars_handler(my_iosys);
+            break;
+        case PIO_MSG_INITDECOMP_DOF:
+            initdecomp_dof_handler(my_iosys);
+            break;
+        case PIO_MSG_WRITEDARRAY:
+            writedarray_handler(my_iosys);
+            break;
+        case PIO_MSG_READDARRAY:
+            readdarray_handler(my_iosys);
+            break;
+        case PIO_MSG_SETERRORHANDLING:
+            seterrorhandling_handler(my_iosys);
+            break;
+        case PIO_MSG_SET_CHUNK_CACHE:
+            set_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_CHUNK_CACHE:
+            get_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_FREEDECOMP:
+            freedecomp_handler(my_iosys);
+            break;
+        case PIO_MSG_SET_FILL:
+            set_fill_handler(my_iosys);
+            break;
+        case PIO_MSG_EXIT:
+            finalize_handler(my_iosys, index);
+            msg = -1;
+            break;
+        default:
+            LOG((0, "unknown message received %d", msg));
+            return PIO_EINVAL;
+        }
+
+        /* If an error was returned by the handler, do nothing! */
+        LOG((3, "pio_msg_handler2 checking error ret = %d", ret));
+
+        /* Listen for another msg from the component whose message we
+         * just handled. */
+        if (!io_rank && msg != -1)
+        {
+            my_iosys = iosys[index];
+            LOG((3, "pio_msg_handler2 about to Irecv index = %d comproot = %d union_comm = %d",
+                 index, my_iosys->comproot, my_iosys->union_comm));
+            if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm,
+                                    &req[index])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "pio_msg_handler2 called MPI_Irecv req[%d] = %d", index, req[index]));
+        }
+
+        LOG((3, "pio_msg_handler2 done msg = %d open_components = %d",
+             msg, open_components));
+
+        /* If there are no more open components, exit. */
+        if (msg == -1)
+            if (--open_components)
+                msg = PIO_MSG_EXIT;
+    }
+
+    LOG((3, "returning from pio_msg_handler2"));
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_nc.c b/src/externals/pio2/src/clib/pio_nc.c
new file mode 100644
index 00000000000..8b7fb81ba64
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_nc.c
@@ -0,0 +1,2763 @@
+/**
+ * @file
+ * PIO interfaces to
+ * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html)
+ * support functions
+ *
+ *  This file provides an interface to the
+ *  [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html)
+ *  support functions.  Each subroutine calls the underlying netcdf or
+ *  pnetcdf or netcdf4 functions from the appropriate subset of mpi
+ *  tasks (io_comm). Each routine must be called collectively from
+ *  union_comm.
+ *
+ * @author Jim Edwards (jedwards@ucar.edu), Ed Hartnett
+ * @date     Feburary 2014, April 2016
+ */
+#include 
+#include 
+#include 
+
+/**
+ * @ingroup PIO_inq
+ * The PIO-C interface for the NetCDF function nc_inq.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm. For more information on the underlying
+ * NetCDF commmand please read about this function in the NetCDF
+ * documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ *
+ * @return PIO_NOERR for success, error code otherwise. See
+ * PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+
+    LOG((1, "PIOc_inq ncid = %d", ncid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ; /* Message for async notification. */
+            char ndims_present = ndimsp ? true : false;
+            char nvars_present = nvarsp ? true : false;
+            char ngatts_present = ngattsp ? true : false;
+            char unlimdimid_present = unlimdimidp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq ncid = %d ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d",
+                 ncid, ndims_present, nvars_present, ngatts_present, unlimdimid_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            LOG((2, "PIOc_inq calling ncmpi_inq unlimdimidp = %d", unlimdimidp));
+            ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);
+            LOG((2, "PIOc_inq called ncmpi_inq"));
+            if (unlimdimidp)
+                LOG((2, "PIOc_inq returned from ncmpi_inq unlimdimid = %d", *unlimdimidp));
+        }
+#endif /* _PNETCDF */
+        if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io)
+        {
+            LOG((2, "PIOc_inq calling classic nc_inq"));
+            /* Should not be necessary to do this - nc_inq should
+             * handle null pointers. This has been reported as a bug
+             * to netCDF developers. */
+            int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid;
+            LOG((2, "PIOc_inq calling classic nc_inq"));
+            ierr = nc_inq(file->fh, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid);
+            LOG((2, "PIOc_inq calling classic nc_inq"));
+            if (unlimdimidp)
+                LOG((2, "classic tmp_unlimdimid = %d", tmp_unlimdimid));
+            if (ndimsp)
+                *ndimsp = tmp_ndims;
+            if (nvarsp)
+                *nvarsp = tmp_nvars;
+            if (ngattsp)
+                *ngattsp = tmp_ngatts;
+            if (unlimdimidp)
+                *unlimdimidp = tmp_unlimdimid;
+            if (unlimdimidp)
+                LOG((2, "classic unlimdimid = %d", *unlimdimidp));
+        }
+        else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            LOG((2, "PIOc_inq calling netcdf-4 nc_inq"));
+            ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);
+        }
+
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (ndimsp)
+        if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (nvarsp)
+        if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (ngattsp)
+        if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (unlimdimidp)
+        if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_ndims
+ * Find out how many dimensions are defined in the file.
+ *
+ * @param ncid the ncid of the open file.
+ * @param ndimsp a pointer that will get the number of dimensions.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_ndims(int ncid, int *ndimsp)
+{
+    LOG((1, "PIOc_inq_ndims"));
+    return PIOc_inq(ncid, ndimsp, NULL, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_nvars
+ * Find out how many variables are defined in a file.
+ *
+ * @param ncid the ncid of the open file.
+ * @param nvarsp a pointer that will get the number of variables.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_nvars(int ncid, int *nvarsp)
+{
+    return PIOc_inq(ncid, NULL, nvarsp, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_natts
+ * Find out how many global attributes are defined in a file.
+ *
+ * @param ncid the ncid of the open file.
+ * @param nattsp a pointer that will get the number of attributes.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_natts(int ncid, int *ngattsp)
+{
+    return PIOc_inq(ncid, NULL, NULL, ngattsp, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_unlimdim
+ * Find out the dimension ids of any unlimited dimensions.
+ *
+ * @param ncid the ncid of the open file.
+ * @param nattsp a pointer that will get an array of unlimited
+ * dimension IDs.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_unlimdim(int ncid, int *unlimdimidp)
+{
+    LOG((1, "PIOc_inq_unlimdim ncid = %d unlimdimidp = %d", ncid, unlimdimidp));
+    return PIOc_inq(ncid, NULL, NULL, NULL, unlimdimidp);
+}
+
+/**
+ * @ingroup PIO_typelen
+ * Learn the name and size of a type.
+ *
+ * @param ncid the ncid of the open file.
+ * @param xtype the type to learn about
+ * @param name pointer that will get the name of the type.
+ * @param sizep pointer that will get the size of the type in bytes.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_type ncid = %d xtype = %d", ncid, xtype));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_TYPE; /* Message for async notification. */
+            char name_present = name ? true : false;
+            char size_present = sizep ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = pioc_pnetcdf_inq_type(ncid, xtype, name, sizep);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_type(file->fh, xtype, name, (size_t *)sizep);
+        LOG((2, "PIOc_inq_type netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (name)
+    {
+        int slen;
+        if (ios->iomaster == MPI_ROOT)
+            slen = strlen(name);
+        if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if (!mpierr)
+            if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+    if (sizep)
+        if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_format
+ * Learn the netCDF format of an open file.
+ *
+ * @param ncid the ncid of an open file.
+ * @param formatp a pointer that will get the format.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_format(int ncid, int *formatp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq ncid = %d", ncid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_FORMAT;
+            char format_present = formatp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_format(file->fh, formatp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_format(file->fh, formatp);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (formatp)
+        if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_dim
+ * The PIO-C interface for the NetCDF function nc_inq_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param lenp a pointer that will get the number of values
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_dim ncid = %d dimid = %d", ncid, dimid));
+
+    /* Get the file info, based on the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_DIM;
+            char name_present = name ? true : false;
+            char len_present = lenp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present));
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            LOG((2, "calling ncmpi_inq_dim"));
+            ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);;
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            LOG((2, "calling nc_inq_dim"));
+            ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
+        }
+        LOG((2, "ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (name)
+    {
+        int slen;
+        LOG((2, "bcasting results my_comm = %d", ios->my_comm));
+        if (ios->iomaster == MPI_ROOT)
+            slen = strlen(name);
+        if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (lenp)
+        if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "done with PIOc_inq_dim"));
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_dimname
+ * Find the name of a dimension.
+ *
+ * @param ncid the ncid of an open file.
+ * @param dimid the dimension ID.
+ * @param name a pointer that gets the name of the dimension. Igorned
+ * if NULL.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_dimname(int ncid, int dimid, char *name)
+{
+    LOG((1, "PIOc_inq_dimname ncid = %d dimid = %d", ncid, dimid));
+    return PIOc_inq_dim(ncid, dimid, name, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_dimlen
+ * Find the length of a dimension.
+ *
+ * @param ncid the ncid of an open file.
+ * @param dimid the dimension ID.
+ * @param lenp a pointer that gets the length of the dimension. Igorned
+ * if NULL.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp)
+{
+    return PIOc_inq_dim(ncid, dimid, NULL, lenp);
+}
+
+/**
+ * @ingroup PIO_inq_dimid
+ * The PIO-C interface for the NetCDF function nc_inq_dimid.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dimid(int ncid, const char *name, int *idp)
+{
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    int ierr;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info, based on the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+    LOG((2, "iosysid = %d", ios->iosysid));
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_dimid ncid = %d name = %s", ncid, name));
+
+    /* If using async, and not an IO task, then send parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_DIMID;
+            char id_present = idp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* IO tasks call the netCDF functions. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_dimid(file->fh, name, idp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_dimid(file->fh, name, idp);
+    }
+    LOG((3, "nc_inq_dimid call complete ierr = %d", ierr));
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (idp)
+        if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_var
+ * The PIO-C interface for the NetCDF function nc_inq_var.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param nattsp a pointer that will get the number of attributes
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
+                 int *dimidsp, int *nattsp)
+{
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    int ndims = 0;    /* The number of dimensions for this variable. */
+    int ierr;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_var ncid = %d varid = %d", ncid, varid));
+
+    /* Get the file info, based on the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+    LOG((2, "got file and iosystem"));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR;
+            char name_present = name ? true : false;
+            char xtype_present = xtypep ? true : false;
+            char ndims_present = ndimsp ? true : false;
+            char dimids_present = dimidsp ? true : false;
+            char natts_present = nattsp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d "
+                 "dimids_present = %d, natts_present = %d nattsp = %d",
+                 name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Call the netCDF layer. */
+    if (ios->ioproc)
+    {
+        LOG((2, "Calling the netCDF layer"));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            ierr = ncmpi_inq_varndims(file->fh, varid, &ndims);
+            LOG((2, "from pnetcdf ndims = %d", ndims));
+            if (!ierr)
+                ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            ierr = nc_inq_varndims(file->fh, varid, &ndims);
+            LOG((2, "file->fh = %d varid = %d xtypep = %d ndimsp = %d dimidsp = %d nattsp = %d",
+                 file->fh, varid, xtypep, ndimsp, dimidsp, nattsp));
+            if (!ierr)
+            {
+                char my_name[NC_MAX_NAME + 1];
+                nc_type my_xtype;
+                int my_ndims = 0, my_dimids[ndims], my_natts = 0;
+                ierr = nc_inq_var(file->fh, varid, my_name, &my_xtype, &my_ndims, my_dimids, &my_natts);
+                LOG((3, "my_name = %s my_xtype = %d my_ndims = %d my_natts = %d",  my_name, my_xtype, my_ndims, my_natts));
+                if (!ierr)
+                {
+                    if (name)
+                        strcpy(name, my_name);
+                    if (xtypep)
+                        *xtypep = my_xtype;
+                    if (ndimsp)
+                        *ndimsp = my_ndims;
+                    if (dimidsp)
+                    {
+                        for (int d = 0; d < ndims; d++)
+                            dimidsp[d] = my_dimids[d];
+                    }
+                    if (nattsp)
+                        *nattsp = my_natts;
+                }
+            }
+        }
+        if (ndimsp)
+            LOG((2, "PIOc_inq_var ndims = %d ierr = %d", *ndimsp, ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast the results for non-null pointers. */
+    if (name)
+    {
+        int slen;
+        if (ios->iomaster == MPI_ROOT)
+            slen = strlen(name);
+        if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+    if (xtypep)
+        if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (ndimsp)
+    {
+        if (ios->ioroot)
+            LOG((2, "PIOc_inq_var about to Bcast ndims = %d ios->ioroot = %d", *ndimsp, ios->ioroot));
+        if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        file->varlist[varid].ndims = *ndimsp;
+        LOG((2, "PIOc_inq_var Bcast ndims = %d", *ndimsp));
+    }
+    if (dimidsp)
+    {
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+    if (nattsp)
+        if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_varname
+ * Get the name of a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param name a pointer that will get the variable name.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_varname(int ncid, int varid, char *name)
+{
+    return PIOc_inq_var(ncid, varid, name, NULL, NULL, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_vartype
+ * Find the type of a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the
+ * attribute. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep)
+{
+    return PIOc_inq_var(ncid, varid, NULL, xtypep, NULL, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_varndims
+ * Find the number of dimensions of a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param ndimsp a pointer that will get the number of
+ * dimensions. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_varndims(int ncid, int varid, int *ndimsp)
+{
+    return PIOc_inq_var(ncid, varid, NULL, NULL, ndimsp, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_vardimid
+ * Find the dimension IDs associated with a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param dimidsp a pointer that will get an array of dimids. Ignored
+ * if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp)
+{
+    return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, dimidsp, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_varnatts
+ * Find the number of attributes associated with a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param nattsp a pointer that will get the number of attriburtes. Ignored
+ * if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_varnatts(int ncid, int varid, int *nattsp)
+{
+    return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, NULL, nattsp);
+}
+
+/**
+ * @ingroup PIO_inq_varid
+ * The PIO-C interface for the NetCDF function nc_inq_varid.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varidp a pointer that will get the variable id
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_varid(int ncid, const char *name, int *varidp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get file info based on ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Caller must provide name. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name));
+
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VARID;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen;
+            namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_varid(file->fh, name, varidp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_varid(file->fh, name, varidp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (varidp)
+        if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_att
+ * The PIO-C interface for the NetCDF function nc_inq_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param lenp a pointer that will get the number of values
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
+                 PIO_Offset *lenp)
+{
+    int msg = PIO_MSG_INQ_ATT;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;
+
+    /* Find file based on ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d",
+         ncid, varid, xtypep, lenp));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            char xtype_present = xtypep ? true : false;
+            char len_present = lenp ? true : false;
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (xtypep)
+        if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (lenp)
+        if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_attlen
+ * Get the length of an attribute.
+ *
+ * @param ncid the ID of an open file.
+ * @param varid the variable ID, or NC_GLOBAL for global attributes.
+ * @param name the name of the attribute.
+ * @param lenp a pointer that gets the lenght of the attribute
+ * array. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp)
+{
+    return PIOc_inq_att(ncid, varid, name, NULL, lenp);
+}
+
+/**
+ * @ingroup PIO_inq_atttype
+ * Get the type of an attribute.
+ *
+ * @param ncid the ID of an open file.
+ * @param varid the variable ID, or NC_GLOBAL for global attributes.
+ * @param name the name of the attribute.
+ * @param xtypep a pointer that gets the type of the
+ * attribute. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep)
+{
+    return PIOc_inq_att(ncid, varid, name, xtypep, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_attname
+ * The PIO-C interface for the NetCDF function nc_inq_attname.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param attnum the attribute ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid,
+         attnum));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_ATTNAME;
+            char name_present = name ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_attname(file->fh, varid, attnum, name);
+        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (name)
+    {
+        int namelen = strlen(name);
+        if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        /* Casting to void to avoid warnings on some compilers. */
+        if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_attid
+ * The PIO-C interface for the NetCDF function nc_inq_attid.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_attid ncid = %d varid = %d name = %s", ncid, varid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_ATTID;
+            int namelen = strlen(name);
+            char id_present = idp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_attid(file->fh, varid, name, idp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_attid(file->fh, varid, name, idp);
+        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (idp)
+        if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_rename_dim
+ * The PIO-C interface for the NetCDF function nc_rename_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_dim(int ncid, int dimid, const char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_rename_dim ncid = %d dimid = %d name = %s", ncid, dimid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_RENAME_DIM; /* Message for async notification. */
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_rename_dim Bcast file->fh = %d dimid = %d namelen = %d name = %s",
+                 file->fh, dimid, namelen, name));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_rename_dim(file->fh, dimid, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_rename_dim(file->fh, dimid, name);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_rename_var
+ * The PIO-C interface for the NetCDF function nc_rename_var.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_var(int ncid, int varid, const char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_rename_var ncid = %d varid = %d name = %s", ncid, varid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_RENAME_VAR; /* Message for async notification. */
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_rename_var Bcast file->fh = %d varid = %d namelen = %d name = %s",
+                 file->fh, varid, namelen, name));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_rename_var(file->fh, varid, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_rename_var(file->fh, varid, name);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_rename_att
+ * The PIO-C interface for the NetCDF function nc_rename_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See
+ * PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_att(int ncid, int varid, const char *name,
+                    const char *newname)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide names of correct length. */
+    if (!name || strlen(name) > NC_MAX_NAME ||
+        !newname || strlen(newname) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_rename_att ncid = %d varid = %d name = %s newname = %s",
+         ncid, varid, name, newname));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_RENAME_ATT; /* Message for async notification. */
+            int namelen = strlen(name);
+            int newnamelen = strlen(newname);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_rename_att(file->fh, varid, name, newname);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_rename_att(file->fh, varid, name, newname);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    LOG((2, "PIOc_rename_att succeeded"));
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_del_att
+ * The PIO-C interface for the NetCDF function nc_del_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name of the attribute to delete.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_del_att(int ncid, int varid, const char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEL_ATT;
+            int namelen = strlen(name); /* Length of name string. */
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_del_att(file->fh, varid, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_del_att(file->fh, varid, name);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_set_fill.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param fillmode either NC_FILL or NC_NOFILL.
+ * @param old_modep a pointer to an int that gets the old setting.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_set_fill
+ */
+int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    LOG((1, "PIOc_set_fill ncid = %d fillmode = %d old_modep = %d", ncid, fillmode,
+         old_modep));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SET_FILL;
+            int old_modep_present = old_modep ? 1 : 0;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_set_fill sent ncid = %d fillmode = %d old_modep_present = %d", ncid, fillmode,
+                 old_modep_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            LOG((3, "about to call ncmpi_set_fill() fillmode = %d", fillmode));
+            ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_set_fill(file->fh, fillmode, old_modep);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (old_modep)
+    {
+        LOG((2, "old_mode = %d", *old_modep));
+        if ((mpierr = MPI_Bcast(old_modep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    LOG((2, "PIOc_set_fill succeeded"));
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_enddef
+ * The PIO-C interface for the NetCDF function nc_enddef.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_enddef(int ncid)
+{
+    return pioc_change_def(ncid, 1);
+}
+
+/**
+ * @ingroup PIO_redef
+ * The PIO-C interface for the NetCDF function nc_redef.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_redef(int ncid)
+{
+    return pioc_change_def(ncid, 0);
+}
+
+/**
+ * @ingroup PIO_def_dim
+ * The PIO-C interface for the NetCDF function nc_def_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_def_dim ncid = %d name = %s len = %d", ncid, name, len));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_DIM;
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+        }
+
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_def_dim(file->fh, name, len, idp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_def_dim(file->fh, name, (size_t)len, idp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (idp)
+        if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "def_dim ierr = %d", ierr));
+    return PIO_NOERR;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_def_var.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varidp a pointer that will get the variable id
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
+                 const int *dimidsp, int *varidp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file information. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name and storage for varid. */
+    if (!name || !varidp || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_def_var ncid = %d name = %s xtype = %d ndims = %d", ncid, name,
+         xtype, ndims));
+
+    /* If using async, and not an IO task, then send parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR;
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);
+#ifdef _NETCDF4
+        /* For netCDF-4 serial files, turn on compression for this variable. */
+        if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C)
+            ierr = nc_def_var_deflate(file->fh, *varidp, 0, 1, 1);
+
+        /* For netCDF-4 parallel files, set parallel access to collective. */
+        if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = nc_var_par_access(file->fh, *varidp, NC_COLLECTIVE);
+#endif /* _NETCDF4 */
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (varidp)
+        if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set the fill value for a variable.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * When the fill mode for the file is NC_FILL, then fill values are
+ * used for missing data. This function sets the fill value to be used
+ * for a variable. If no specific fill value is set (as a _FillValue
+ * attribute), then the default fill values from netcdf.h are used.
+ *
+ * NetCDF-4 and pnetcdf files allow setting fill_mode (to NC_FILL or
+ * NC_NOFILL) on a per-variable basis. NetCDF classic only allows the
+ * fill_mode setting to be set for the whole file. For this function,
+ * the fill_mode parameter is ignored for classic files. Set the
+ * file-level fill mode with PIOc_set_fill().
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param fill_mode fill mode for this variable (NC_FILL or NC_NOFILL)
+ * @param fill_value pointer to the fill value to be used if fill_mode is set to NC_FILL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_valuep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    nc_type xtype;         /* The type of the variable (and fill value att). */
+    PIO_Offset type_size;  /* Size in bytes of this variable's type. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_def_var_fill ncid = %d varid = %d fill_mode = %d\n", ncid, varid,
+         fill_mode));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Caller must provide correct values. */
+    if ((fill_mode != NC_FILL && fill_mode != NC_NOFILL) ||
+        (fill_mode == NC_FILL && !fill_valuep))
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. Get the size of this vars
+     * type. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &type_size)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+    }
+    LOG((2, "PIOc_def_var_fill type_size = %d", type_size));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_FILL;
+            char fill_value_present = fill_valuep ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && fill_value_present)
+                mpierr = MPI_Bcast((PIO_Offset *)fill_valuep, type_size, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_def_var_fill ncid = %d varid = %d fill_mode = %d type_size = %d fill_value_present = %d",
+                 ncid, varid, fill_mode, type_size, fill_value_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+#ifdef _PNETCDF
+            ierr = ncmpi_def_var_fill(file->fh, varid, fill_mode, (void *)fill_valuep);
+#endif /* _PNETCDF */
+        }
+        else if (file->iotype == PIO_IOTYPE_NETCDF)
+        {
+            LOG((2, "defining fill value attribute for netCDF classic file"));
+            if (file->do_io)            
+                ierr = nc_put_att(file->fh, varid, _FillValue, xtype, 1, fill_valuep);
+        }
+        else
+        {
+#ifdef _NETCDF4
+            if (file->do_io)
+                ierr = nc_def_var_fill(file->fh, varid, fill_mode, fill_valuep);
+#endif
+        }
+        LOG((2, "after def_var_fill ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_inq_var_fill.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param fill_mode a pointer to int that will get the fill
+ * mode. Ignored if NULL (except with pnetcdf, which seg-faults with
+ * NULL.)
+ * @param fill_valuep pointer to space that gets the fill value for
+ * this variable. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_fill
+ */
+int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    nc_type xtype;         /* Type of variable and its _FillValue attribute. */
+    PIO_Offset type_size;  /* Size in bytes of this variable's type. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+
+    LOG((1, "PIOc_inq_var_fill ncid = %d varid = %d", ncid, varid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+    LOG((2, "found file"));
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. Get the size of this vars
+     * type. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &type_size)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_inq_var_fill type_size = %d", type_size));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_FILL;
+            char fill_mode_present = fill_mode ? true : false;
+            char fill_value_present = fill_valuep ? true : false;
+
+            LOG((2, "sending msg type_size = %d", type_size));
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_mode_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld fill_mode_present = %d fill_value_present = %d",
+                 ncid, varid, type_size, fill_mode_present, fill_value_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        LOG((2, "calling inq_var_fill file->iotype = %d file->fh = %d varid = %d fill_mode = %d",
+             file->iotype, file->fh, varid, fill_mode));
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+#ifdef _PNETCDF
+            ierr = ncmpi_inq_var_fill(file->fh, varid, fill_mode, fill_valuep);
+#endif /* _PNETCDF */
+        }
+        else if (file->iotype == PIO_IOTYPE_NETCDF)
+        {
+            /* Get the file-level fill mode. */
+            if (fill_mode)
+            {
+                ierr = nc_set_fill(file->fh, NC_NOFILL, fill_mode);
+                if (!ierr)
+                    ierr = nc_set_fill(file->fh, *fill_mode, NULL);
+            }
+
+            if (!ierr && fill_valuep)
+            {
+                ierr = nc_get_att(file->fh, varid, _FillValue, fill_valuep);
+                if (ierr == NC_ENOTATT)
+                {
+                    char char_fill_value = NC_FILL_CHAR;
+                    signed char byte_fill_value = NC_FILL_BYTE;
+                    short short_fill_value = NC_FILL_SHORT;
+                    int int_fill_value = NC_FILL_INT;
+                    float float_fill_value = NC_FILL_FLOAT;
+                    double double_fill_value = NC_FILL_DOUBLE;
+                    switch (xtype)
+                    {
+                    case NC_BYTE:
+                        memcpy(fill_valuep, &byte_fill_value, sizeof(signed char));
+                        break;
+                    case NC_CHAR:
+                        memcpy(fill_valuep, &char_fill_value, sizeof(char));
+                        break;
+                    case NC_SHORT:
+                        memcpy(fill_valuep, &short_fill_value, sizeof(short));
+                        break;
+                    case NC_INT:
+                        memcpy(fill_valuep, &int_fill_value, sizeof(int));
+                        break;
+                    case NC_FLOAT:
+                        memcpy(fill_valuep, &float_fill_value, sizeof(float));
+                        break;
+                    case NC_DOUBLE:
+                        memcpy(fill_valuep, &double_fill_value, sizeof(double));
+                        break;
+                    default:
+                        return pio_err(ios, file, NC_EBADTYPE, __FILE__, __LINE__);
+                    }
+                    ierr = PIO_NOERR;
+                }
+            }
+        }
+        else
+        {
+#ifdef _NETCDF4
+            /* The inq_var_fill is not supported in classic-only builds. */
+            if (file->do_io)
+                ierr = nc_inq_var_fill(file->fh, varid, fill_mode, fill_valuep);
+#endif /* _NETCDF */
+        }
+        LOG((2, "after call to inq_var_fill, ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (fill_mode)
+        if ((mpierr = MPI_Bcast(fill_mode, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (fill_valuep)
+        if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the value of an attribute of any type, with no type conversion.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_get_att
+ */
+int PIOc_get_att(int ncid, int varid, const char *name, void *ip)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    nc_type atttype;       /* The type of the attribute. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a name and destination pointer. */
+    if (!name || !ip || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_get_att ncid %d varid %d name %s", ncid, varid, name));
+
+    /* Get the type of the attribute. */
+    if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, NULL)))
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((2, "atttype = %d", atttype));
+
+    return PIOc_get_att_tc(ncid, varid, name, atttype, ip);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute of any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype,
+                 PIO_Offset len, const void *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, xtype, op);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 64-bit floating point array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_DOUBLE, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 8-bit unsigned char array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_UBYTE, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 16-bit unsigned integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_USHORT, (void *)ip);
+}
+
+/**
+ * Get the value of an 32-bit unsigned integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_get_att
+ */
+int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_UINT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 32-bit ingeger array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_LONG_INTERNAL, (void *)ip);
+}
+
+/**
+ * Get the value of an text attribute. There is no type conversion
+ * with this call. If the attribute is not of type NC_CHAR, then an
+ * error will be returned.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_get_att
+ */
+int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_CHAR, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 8-bit signed char array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_BYTE, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 64-bit unsigned integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_UINT64, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 16-bit integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_SHORT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 32-bit integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_INT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 64-bit integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_INT64, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 32-bit floating point array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_FLOAT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 8-bit signed chars.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const signed char *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_BYTE, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit signed integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype,
+                      PIO_Offset len, const long *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit signed integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype,
+                     PIO_Offset len, const int *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_INT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 8-bit unsigned chars.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const unsigned char *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UBYTE, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 64-bit signed integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype,
+                          PIO_Offset len, const long long *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_INT64, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit unsigned integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype,
+                      PIO_Offset len, const unsigned int *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UINT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit floating points.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const float *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_FLOAT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 64-bit unsigned integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
+                           PIO_Offset len, const unsigned long long *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UINT64, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 16-bit unsigned integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype,
+                        PIO_Offset len, const unsigned short *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_USHORT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF text attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_text(int ncid, int varid, const char *name,
+                      PIO_Offset len, const char *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, NC_CHAR, len, NC_CHAR, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 16-bit integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const short *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_SHORT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 64-bit floating points.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype,
+                        PIO_Offset len, const double *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_DOUBLE, op);
+}
diff --git a/src/externals/pio2/src/clib/pio_nc4.c b/src/externals/pio2/src/clib/pio_nc4.c
new file mode 100644
index 00000000000..b68bba41050
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_nc4.c
@@ -0,0 +1,1031 @@
+/** @file
+ *
+ * Functions to wrap netCDF-4 functions for PIO.
+ *
+ * @author Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/**
+ * Set deflate (zlib) settings for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param shuffle non-zero to turn on shuffle filter (can be good for
+ * integer data).
+ * @param deflate non-zero to turn on zlib compression for this
+ * variable.
+ * @param deflate_level 1 to 9, with 1 being faster and 9 being more
+ * compressed.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
+                         int deflate_level)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_DEFLATE;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors from computation tasks. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = NC_EINVAL;
+        else
+            if (file->do_io)
+                ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Inquire about deflate (zlib compression) settings for a variable.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param shufflep pointer to an int that will get the status of the
+ * shuffle filter. Ignored if NULL.
+ * @param deflatep pointer to an int that will be set to non-zero if
+ * deflation is in use for this variable. Ignored if NULL.
+ * @param deflate_levelp pointer to an int that will get the deflation
+ * level (from 1-9) if deflation is in use for this variable.  Ignored
+ * if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
+                         int *deflate_levelp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_DEFLATE;
+            char shuffle_present = shufflep ? true : false;
+            char deflate_present = deflatep ? true : false;
+            char deflate_level_present = deflate_levelp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (shuffle_present && !mpierr)
+                mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (deflate_present && !mpierr)
+                mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (deflate_level_present && !mpierr)
+                mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var_deflate ncid = %d varid = %d shuffle_present = %d deflate_present = %d "
+                 "deflate_level_present = %d", ncid, varid, shuffle_present, deflate_present,
+                 deflate_level_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (shufflep)
+        if ((mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (deflatep)
+        if ((mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (deflate_levelp)
+        if ((mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_def_var
+ * Set chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storage NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * every variable dimension.
+ * @return PIO_NOERR for success, otherwise an error code.
+ */
+int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;             /* The number of dimensions for this var. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_def_var_chunking ncid = %d varid = %d storage = %d", ncid,
+         varid, storage));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. Get the number of
+     * dimensions. */
+    if (!ios->async_interface || !ios->ioproc)
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((2, "PIOc_def_var_chunking first ndims = %d", ndims));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_CHUNKING;
+            char chunksizes_present = chunksizesp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&storage, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && chunksizes_present)
+                mpierr = MPI_Bcast((PIO_Offset *)chunksizesp, ndims, MPI_OFFSET, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_def_var_chunking ncid = %d varid = %d storage = %d ndims = %d chunksizes_present = %d",
+                 ncid, varid, storage, ndims, chunksizes_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    LOG((2, "PIOc_def_var_chunking ndims = %d", ndims));
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+        {
+            size_t chunksizes_sizet[ndims];
+            for (int d = 0; d < ndims; d++)
+            {
+                if (chunksizesp[d] < 0)
+                {
+                    ierr = PIO_ERANGE;
+                    break;
+                }
+                chunksizes_sizet[d] = chunksizesp[d];
+            }
+            if (!ierr)
+                ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizes_sizet);
+        }
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Inquire about chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storagep pointer to int which will be set to either
+ * NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep pointer to memory where chunksizes will be
+ * set. There are the same number of chunksizes as there are
+ * dimensions.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ndims; /* The number of dimensions in the variable. */
+
+    LOG((1, "PIOc_inq_var_chunking ncid = %d varid = %d"));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async_interface || !ios->ioproc)
+    {
+        /* Find the number of dimensions of this variable. */
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        LOG((2, "ndims = %d", ndims));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_CHUNKING;
+            char storage_present = storagep ? true : false;
+            char chunksizes_present = chunksizesp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var_chunking ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
+                 ncid, varid, storage_present, chunksizes_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+        {
+            size_t chunksizes_sizet[ndims];
+            ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizes_sizet);
+            if (!ierr && chunksizesp)
+                for (int d = 0; d < ndims; d++)
+                {
+                    if (chunksizes_sizet[d] > NC_MAX_INT64)
+                    {
+                        ierr = PIO_ERANGE;
+                        break;
+                    }
+                    chunksizesp[d] = chunksizes_sizet[d];
+                }
+        }
+#endif
+        LOG((2, "ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (storagep)
+        if ((mpierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (chunksizesp)
+        if ((mpierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storage NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * every variable dimension.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var_endian(int ncid, int varid, int endian)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_ENDIAN;
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&endian, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_def_var_endian(file->fh, varid, endian);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Inquire about chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param endianp pointer to int which will be set to
+ * endianness. Ignored if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_var_endian ncid = %d varid = %d"));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_ENDIAN;
+            char endian_present = endianp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_inq_var_endian(file->fh, varid, endianp);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (endianp)
+        if ((mpierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set chunk cache netCDF files to be opened/created.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * The file chunk cache for HDF5 can be set, and will apply for any
+ * files opened or created until the program ends, or the settings are
+ * changed again. The cache settings apply only to the open file. They
+ * do not persist with the file, and must be set each time the file is
+ * opened, before it is opened, if they are to have effect.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param iotype the iotype of files to be created or opened.
+ * @param size size of file cache.
+ * @param nelems number of elements in file cache.
+ * @param preemption preemption setting for file cache.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
+                         float preemption)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_set_chunk_cache iosysid = %d iotype = %d size = %d nelems = %d preemption = %g",
+         iosysid, iotype, size, nelems, preemption));
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Only netCDF-4 files can use this feature. */
+    if (iotype != PIO_IOTYPE_NETCDF4P && iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SET_CHUNK_CACHE; /* Message for async notification. */
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        LOG((2, "calling nc_chunk_cache"));
+        if (iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = nc_set_chunk_cache(size, nelems, preemption);
+        else
+            if (!ios->io_rank)
+                ierr = nc_set_chunk_cache(size, nelems, preemption);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+    LOG((2, "PIOc_set_chunk_cache complete!"));
+    return PIO_NOERR;
+}
+
+/**
+ * Get current file chunk cache settings from HDF5.
+ *
+ * This function has no effect on netCDF classic files. Calling this
+ * function with iotype of PIO_IOTYPE_PNETCDF or PIO_IOTYPE_NETCDF
+ * returns an error.
+ *
+ * The file chunk cache for HDF5 can be set, and will apply for any
+ * files opened or created until the program ends, or the settings are
+ * changed again. The cache settings apply only to the open file. They
+ * do not persist with the file, and must be set each time the file is
+ * opened, before it is opened, if they are to have effect.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * @param iotype the iotype of files to be created or opened.
+ * @param sizep gets the size of file cache.
+ * @param nelemsp gets the number of elements in file cache.
+ * @param preemptionp gets the preemption setting for file cache.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                         float *preemptionp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_get_chunk_cache iosysid = %d iotype = %d", iosysid, iotype));
+
+    /* Get the io system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Only netCDF-4 files can use this feature. */
+    if (iotype != PIO_IOTYPE_NETCDF4P && iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_CHUNK_CACHE; /* Message for async notification. */
+            char size_present = sizep ? true : false;
+            char nelems_present = nelemsp ? true : false;
+            char preemption_present = preemptionp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_get_chunk_cache size_present = %d nelems_present = %d "
+                 "preemption_present = %d ", size_present, nelems_present, preemption_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
+        else
+            if (!ios->io_rank)
+                ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
+#endif
+        LOG((2, "nc_get_chunk_cache called ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "bcast complete ierr = %d sizep = %d", ierr, sizep));
+    if (ierr)
+        return check_netcdf(NULL, ierr, __FILE__, __LINE__);
+
+    if (sizep)
+    {
+        LOG((2, "bcasting size = %d ios->ioroot = %d", *sizep, ios->ioroot));
+        if ((mpierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((2, "bcast size = %d", *sizep));
+    }
+    if (nelemsp)
+    {
+        if ((mpierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((2, "bcast complete nelems = %d", *nelemsp));
+    }
+    if (preemptionp)
+    {
+        if ((mpierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((2, "bcast complete preemption = %d", *preemptionp));
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storage NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * every variable dimension.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
+                             float preemption)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SET_VAR_CHUNK_CACHE;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the variable chunk cache settings.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param sizep will get the size of the cache in bytes. Ignored if NULL.
+ * @param nelemsp will get the number of elements in the cache. Ignored if NULL.
+ * @param preemptionp will get the cache preemption value. Ignored if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                             float *preemptionp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_get_var_chunk_cache ncid = %d varid = %d"));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_VAR_CHUNK_CACHE; /* Message for async notification. */
+            char size_present = sizep ? true : false;
+            char nelems_present = nelemsp ? true : false;
+            char preemption_present = preemptionp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_get_var_chunk_cache size_present = %d nelems_present = %d "
+                 "preemption_present = %d ", size_present, nelems_present, preemption_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_get_var_chunk_cache(file->fh, varid, (size_t *)sizep, (size_t *)nelemsp,
+                                          preemptionp);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (sizep && !ierr)
+        if ((mpierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (nelemsp && !ierr)
+        if ((mpierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (preemptionp && !ierr)
+        if ((mpierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_put_nc.c b/src/externals/pio2/src/clib/pio_put_nc.c
new file mode 100644
index 00000000000..74279c9871e
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_put_nc.c
@@ -0,0 +1,1190 @@
+/**
+ * @file
+ * PIO functions to write data.
+ *
+ * @author Ed Hartnett
+ * @date  2016
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/**
+ * Get strided, muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const char *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride,
+                        const unsigned char *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                        const PIO_Offset *stride, const signed char *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, const unsigned short *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, const short *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const unsigned int *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const int *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const long *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                        const PIO_Offset *stride, const float *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit unsigned integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const long long *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit floating point
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, const double *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 64-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const unsigned long long *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op);
+}
+
+/**
+ * Get one value from an text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_CHAR, op);
+}
+
+/**
+ * Get one value from an text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
+                        const unsigned char *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_UBYTE, op);
+}
+
+/**
+ * Get one value from an signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index,
+                        const signed char *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_BYTE, op);
+}
+
+/**
+ * Get one value from an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
+                         const unsigned short *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_USHORT, op);
+}
+
+/**
+ * Get one value from a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index,
+                        const short *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_SHORT, op);
+}
+
+/**
+ * Get one value from an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
+                       const unsigned int *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_UINT, op);
+}
+
+/**
+ * Get one value from an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_INT, op);
+}
+
+/**
+ * Get one value from an floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_FLOAT, op);
+}
+
+/**
+ * Get one value from an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * Get one value from an 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index,
+                         const double *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_DOUBLE, op);
+}
+
+/**
+ * Get one value from an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
+                            const unsigned long long *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_UINT64, op);
+}
+
+/**
+ * Get one value from a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index,
+                           const long long *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_INT64, op);
+}
+
+/**
+ * Put muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const char *op)
+{
+    return PIOc_put_vars_text(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const unsigned char *op)
+{
+    return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const signed char *op)
+{
+    return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const unsigned short *op)
+{
+    return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const short *op)
+{
+    return PIOc_put_vars_short(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const unsigned int *op)
+{
+    return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start,
+                      const PIO_Offset *count, const int *op)
+{
+    return PIOc_put_vars_int(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const long *op)
+{
+    return PIOc_put_vars_long(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const float *op)
+{
+    return PIOc_put_vars_float(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const unsigned long long *op)
+{
+    return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const long long *op)
+{
+    return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const double *op)
+{
+    return PIOc_put_vars_double(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put all data to a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_text(int ncid, int varid, const char *op)
+{
+    return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op)
+{
+    return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_schar(int ncid, int varid, const signed char *op)
+{
+    return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to a 16-bit unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op);
+}
+
+/**
+ * Put all data to a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_short(int ncid, int varid, const short *op)
+{
+    return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op)
+{
+    return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_int(int ncid, int varid, const int *op)
+{
+    return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_long(int ncid, int varid, const long *op)
+{
+    return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_float(int ncid, int varid, const float *op)
+{
+    return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op)
+{
+    return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_longlong(int ncid, int varid, const long long *op)
+{
+    return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_double(int ncid, int varid, const double *op)
+{
+    return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op);
+}
+
+/**
+ * Put all data to a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var(int ncid, int varid, const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_NAT, op);
+}
+
+/**
+ * Get one value from a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_NAT, op);
+}
+
+/**
+ * Put muti-dimensional subset of a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_NAT, op);
+}
+
+/**
+ * Write strided, muti-dimensional subset of a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const PIO_Offset *stride, const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_NAT, op);
+}
diff --git a/src/externals/pio2/src/clib/pio_rearrange.c b/src/externals/pio2/src/clib/pio_rearrange.c
new file mode 100644
index 00000000000..458e7e3844f
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_rearrange.c
@@ -0,0 +1,1900 @@
+/** @file
+ * Code to map IO to model decomposition.
+ *
+ * @author Jim Edwards
+ */
+#include 
+#include 
+#include 
+
+/**
+ * Internal library util function to initialize rearranger options.
+ *
+ * @param iosys pointer to iosystem descriptor
+ */
+void init_rearr_opts(iosystem_desc_t *iosys)
+{
+    /* The old default for max pending requests was 64 - we no longer use it*/
+
+    /* Disable handshake /isend and set max_pend_req = 0 to turn of throttling */
+    const rearr_comm_fc_opt_t def_coll_comm_fc_opts = { false, false, 0 };
+
+    assert(iosys);
+
+    /* Default to coll - i.e., no flow control */
+    iosys->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    iosys->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+    iosys->rearr_opts.comm_fc_opts_comp2io = def_coll_comm_fc_opts;
+    iosys->rearr_opts.comm_fc_opts_io2comp = def_coll_comm_fc_opts;
+}
+
+/**
+ * Convert an index into a list of dimensions. E.g., for index 4 into a
+ * array defined as a[3][2], will return 1 1.
+ *
+ * @param ndims number of dimensions
+ * @param gdims
+ * @param idx
+ * @param dim_list
+ * @returns 0 on success, error code otherwise.
+ */
+void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx,
+                     PIO_Offset *dim_list)
+{
+    int i, curr_idx, next_idx;
+    curr_idx = idx;
+
+    /* Easiest to start from the right and move left. */
+    for (i = ndims - 1; i >= 0; --i)
+    {
+        /* This way of doing div/mod is slightly faster than using "/"
+         * and "%". */
+        next_idx = curr_idx / gdims[i];
+        dim_list[i] = curr_idx - (next_idx * gdims[i]);
+        curr_idx = next_idx;
+    }
+}
+
+/**
+ * Expand a region along dimension dim, by incrementing count[i] as
+ * much as possible, consistent with the map.
+ *
+ * Once max_size is reached, the map is exhausted, or the next entries fail
+ * to match, expand_region updates the count and calls itself with the next
+ * outermost dimension, until the region has been expanded as much as
+ * possible along all dimensions.
+ *
+ * @param dim
+ * @param gdims array of dimension ids
+ * @param maplen the length of the map
+ * @param map
+ * @param region_size
+ * @param region_stride
+ * @param max_size array of maximum sizes
+ * @param count array of counts
+ * @returns 0 on success, error code otherwise.
+ */
+void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map,
+                   int region_size, int region_stride, const int *max_size,
+                   PIO_Offset *count)
+{
+    int i, j, test_idx, expansion_done;
+    /* Precondition: maplen >= region_size (thus loop runs at least
+     * once). */
+
+    /* Flag used to signal that we can no longer expand the region
+       along dimension dim. */
+    expansion_done = 0;
+
+    /* Expand no greater than max_size along this dimension. */
+    for (i = 1; i <= max_size[dim]; ++i)
+    {
+        /* Count so far is at least i. */
+        count[dim] = i;
+
+        /* Now see if we can expand to i+1 by checking that the next
+           region_size elements are ahead by exactly region_stride.
+           Assuming monotonicity in the map, we could skip this for the
+           innermost dimension, but it's necessary past that because the
+           region does not necessarily comprise contiguous values. */
+        for (j = 0; j < region_size; ++j)
+        {
+            test_idx = j + i * region_size;
+
+            /* If we have exhausted the map, or the map no longer matches,
+               we are done, break out of both loops. */
+            if (test_idx >= maplen || map[test_idx] != map[j] + i * region_stride)
+            {
+                expansion_done = 1;
+                break;
+            }
+        }
+        if (expansion_done)
+            break;
+    }
+
+    /* Move on to next outermost dimension if there are more left,
+     * else return. */
+    if (dim > 0)
+        expand_region(dim-1, gdims, maplen, map, region_size * count[dim],
+                      region_stride * gdims[dim], max_size, count);
+}
+
+/**
+ * Set start and count so that they describe the first region in map.
+ *
+ * @param ndims the number of dimensions
+ * @param gdims pointer to an array of dimension ids
+ * @param maplen the length of the map
+ * @param map
+ * @param start array of start indicies
+ * @param count array of counts
+ * @returns 0 on success, error code otherwise.
+ */
+PIO_Offset find_region(int ndims, const int *gdims, int maplen, const PIO_Offset *map,
+                       PIO_Offset *start, PIO_Offset *count)
+{
+    int dim;
+    int max_size[ndims];
+    PIO_Offset regionlen = 1;
+
+    /* Preconditions (which might be useful to check/assert):
+       ndims is > 0
+       maplen is > 0
+       all elements of map are inside the bounds specified by gdims
+       The map array is 1 based, but calculations are 0 based */
+    idx_to_dim_list(ndims, gdims, map[0] - 1, start);
+
+    /* Can't expand beyond the array edge.*/
+    for (dim = 0; dim < ndims; ++dim)
+        max_size[dim] = gdims[dim] - start[dim];
+
+    /* For each dimension, figure out how far we can expand in that dimension
+       while staying contiguous in the input array.
+
+       Start with the innermost dimension (ndims-1), and it will recurse
+       through to the outermost dimensions. */
+    expand_region(ndims - 1, gdims, maplen, map, 1, 1, max_size, count);
+
+    for (dim = 0; dim < ndims; dim++)
+        regionlen *= count[dim];
+
+    return(regionlen);
+}
+
+/**
+ * Convert a global coordinate value into a local array index.
+ *
+ * @param ndims the number of dimensions
+ * @param lcoord pointer to an offset
+ * @param count array of counts
+ * @returns 0 on success, error code otherwise.
+ */
+PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count)
+{
+    PIO_Offset lindex = 0;
+    PIO_Offset stride = 1;
+
+    for (int i = ndims - 1; i >= 0; i--)
+    {
+        lindex += lcoord[i] * stride;
+        stride = stride * count[i];
+    }
+    return lindex;
+}
+
+/**
+ * Compute the max io buffersize needed for a given variable
+ *
+ * @param io_comm the IO communicator
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 for success, error code otherwise.
+ */
+int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc)
+{
+    PIO_Offset iosize, totiosize;
+    int i;
+    io_region *region;
+    int mpierr; /* Return code from MPI calls. */
+
+    assert(iodesc);
+
+    /*  compute the max io buffer size, for conveneance it is the
+     *  combined size of all regions */
+    totiosize = 0;
+    region = iodesc->firstregion;
+    while(region)
+    {
+        if (region->count[0] > 0)
+        {
+            iosize = 1;
+            for (i = 0; i < iodesc->ndims; i++)
+                iosize *= region->count[i];
+            totiosize += iosize;
+        }
+        region = region->next;
+    }
+
+    /* Share the max io buffer size with all io tasks. */
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totiosize, 1, MPI_OFFSET, MPI_MAX, io_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    iodesc->maxiobuflen = totiosize;
+
+    if (iodesc->maxiobuflen <= 0)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Create the derived MPI datatypes used for comp2io and io2comp
+ * transfers. Used in define_iodesc_datatypes().
+ *
+ * @param basetype The type of data (int,real,double).
+ * @param msgcnt The number of MPI messages/tasks to use.
+ * @param dlen The length of the data array.
+ * @param mindex An array of indexes into the data array from the comp
+ * map
+ * @param mcount The number of indexes to be put on each mpi
+ * message/task
+ * @param mfrom A pointer to the previous structure in the read/write
+ * list
+ * @param mtype The final data structure sent through MPI to the
+ * read/write
+ * @returns 0 on success, error code otherwise.
+ */
+int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen,
+                         const PIO_Offset *mindex, const int *mcount, int *mfrom,
+                         MPI_Datatype *mtype)
+{
+    PIO_Offset bsizeT[msgcnt];
+    int blocksize;
+    int numinds = 0;
+    PIO_Offset *lindex = NULL;
+    int mpierr; /* Return code from MPI functions. */
+
+    for (int j = 0; j < msgcnt; j++)
+        numinds += mcount[j];
+
+    pioassert(dlen >= 0, "dlen < 0", __FILE__, __LINE__);
+    pioassert(numinds >= 0, "num inds < 0", __FILE__, __LINE__);
+
+    if (mindex)
+    {
+        /* memcpy(lindex, mindex, (size_t) (dlen*sizeof(PIO_Offset)));*/
+        if (!(lindex = malloc(numinds * sizeof(PIO_Offset))))
+            return PIO_ENOMEM;
+        memcpy(lindex, mindex, (size_t)(numinds * sizeof(PIO_Offset)));
+    }
+
+    bsizeT[0] = 0;
+    mtype[0] = PIO_DATATYPE_NULL;
+    int pos = 0;
+    int ii = 0;
+
+    /* If there are no messages don't need to create any datatypes. */
+    if (msgcnt > 0)
+    {
+        if (mfrom == NULL)
+        {
+            for (int i = 0; i < msgcnt; i++)
+            {
+                if (mcount[i] > 0)
+                {
+                    bsizeT[ii] = GCDblocksize(mcount[i], lindex+pos);
+                    ii++;
+                    pos += mcount[i];
+                }
+            }
+            blocksize = (int)lgcd_array(ii ,bsizeT);
+        }
+        else
+        {
+            blocksize=1;
+        }
+
+        /* pos is an index to the start of each message block. */
+        pos = 0;
+        for (int i = 0; i < msgcnt; i++)
+        {
+            if (mcount[i] > 0)
+            {
+                int len = mcount[i] / blocksize;
+                int displace[len];
+                if (blocksize == 1)
+                {
+                    if (!mfrom)
+                    {
+                        for (int j = 0; j < len; j++)
+                            displace[j] = (int)(lindex[pos + j]);
+                    }
+                    else
+                    {
+                        int k = 0;
+                        for (int j = 0; j < numinds; j++)
+                            if (mfrom[j] == i)
+                                displace[k++] = (int)(lindex[j]);
+                    }
+
+                }
+                else
+                {
+                    for (int j = 0; j < mcount[i]; j++)
+                        (lindex + pos)[j]++;
+
+                    for (int j = 0; j < len; j++)
+                        displace[j] = ((lindex + pos)[j * blocksize] - 1);
+                }
+
+                if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace, basetype, mtype + i)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                if (mtype[i] == PIO_DATATYPE_NULL)
+                    return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+                /* Commit the MPI data type. */
+                if ((mpierr = MPI_Type_commit(mtype + i)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                pos += mcount[i];
+            }
+        }
+        if (lindex)
+            free(lindex);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Create the derived MPI datatypes used for comp2io and io2comp
+ * transfers.
+ *
+ * NOTE from Jim: I am always oriented toward write so recieve
+ * always means io tasks and send always means comp tasks. The
+ * opposite relationship is actually the case for reading. I've
+ * played with different ways of referring to things to get rid of
+ * this orientation bias in the documentation as well as in
+ * variable names, but I haven't found anything that I found more
+ * satisfactory.
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int ret; /* Return value. */
+
+    /* Set up the to transfer data to and from the IO tasks. */
+    if (ios->ioproc)
+    {
+        if (!iodesc->rtype)
+        {
+            if (iodesc->nrecvs > 0)
+            {
+                /* Allocate memory for array of data. */
+                if (!(iodesc->rtype = malloc(iodesc->nrecvs * sizeof(MPI_Datatype))))
+                    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+                /* Initialize data types to NULL. */
+                for (int i = 0; i < iodesc->nrecvs; i++)
+                    iodesc->rtype[i] = PIO_DATATYPE_NULL;
+
+                /* Create the datatypes, which will be used both to
+                 * receive and to send data. */
+                if (iodesc->rearranger == PIO_REARR_SUBSET)
+                {
+                    if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen,
+                                                    iodesc->rindex, iodesc->rcount, iodesc->rfrom,
+                                                    iodesc->rtype)))
+                        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+                }
+                else
+                {
+                    if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen,
+                                                    iodesc->rindex, iodesc->rcount, NULL, iodesc->rtype)))
+                        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+                }
+            }
+        }
+    }
+
+    /* Define the send datatypes if they don't exist. */
+    if (!iodesc->stype)
+    {
+        int ntypes;
+        int ncnt;
+
+        if (iodesc->rearranger == PIO_REARR_SUBSET)
+        {
+            /* Subset rearranger gets one type. */
+            ntypes = 1;
+            ncnt = iodesc->scount[0];
+        }
+        else
+        {
+            /* Box rearranger gets one type per IO task. */
+            ntypes = ios->num_iotasks;
+            ncnt = iodesc->ndof;
+        }
+
+        /* Allocate memory for array of send types. */
+        if (!(iodesc->stype = malloc(ntypes * sizeof(MPI_Datatype))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Initialize send types to NULL. */
+        for (int i = 0; i < ntypes; i++)
+            iodesc->stype[i] = PIO_DATATYPE_NULL;
+
+        iodesc->num_stypes = ntypes;
+
+        /* Create the MPI data types. */
+        if ((ret = create_mpi_datatypes(iodesc->basetype, ntypes, ncnt, iodesc->sindex,
+                                        iodesc->scount, NULL, iodesc->stype)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ *  Completes the mapping for the box rearranger.
+ *
+ * @param ios pointer to the iosystem_desc_t struct
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param maplen the length of the map
+ * @param dest_ioproc an array of IO task numbers
+ * @param dest_ioindex
+ * @param mycomm an MPI communicator
+ * @returns 0 on success, error code otherwise.
+ */
+int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen,
+                   const int *dest_ioproc, const PIO_Offset *dest_ioindex, MPI_Comm mycomm)
+{
+    int i;
+    int iorank;
+    int rank;
+    int ntasks;
+    int mpierr; /* Return call from MPI functions. */
+
+    /* Find size of communicator, and task rank. */
+    if ((mpierr = MPI_Comm_rank(mycomm, &rank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    MPI_Datatype sr_types[ntasks];
+    int send_counts[ntasks];
+    int send_displs[ntasks];
+    int recv_counts[ntasks];
+    int recv_displs[ntasks];
+    int *recv_buf = NULL;
+    int nrecvs;
+    int ierr;
+    int io_comprank;
+    int ioindex;
+    int tsize;
+    int numiotasks;
+    PIO_Offset s2rindex[iodesc->ndof];
+
+    pioassert(iodesc, "iodesc must be provided", __FILE__, __LINE__);
+
+    /* Subset rearranger always gets 1 IO task. */
+    if (iodesc->rearranger == PIO_REARR_BOX)
+        numiotasks = ios->num_iotasks;
+    else
+        numiotasks = 1;
+
+    /* Allocate memory for the array of counts. */
+    if (!(iodesc->scount = malloc(numiotasks * sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Initialize counts to zero. */
+    for (i = 0; i < numiotasks; i++)
+        iodesc->scount[i] = 0;
+
+    /* iodesc->scount is the amount of data sent to each task from the
+     * current task */
+    for (i = 0; i < maplen; i++)
+        if (dest_ioindex[i] >= 0)
+            (iodesc->scount[dest_ioproc[i]])++;
+
+    /* Initialize arrays. */
+    for (i = 0; i < ntasks; i++)
+    {
+        send_counts[i] = 0;
+        send_displs[i] = 0;
+        recv_counts[i] = 0;
+        recv_displs[i] = 0;
+        sr_types[i] = MPI_INT;
+    }
+
+    /* ??? */
+    for (i = 0; i < numiotasks; i++)
+    {
+        int io_comprank;
+        if (iodesc->rearranger == PIO_REARR_SUBSET)
+            io_comprank = 0;
+        else
+            io_comprank = ios->ioranks[i];
+        send_counts[io_comprank] = 1;
+        send_displs[io_comprank] = i * sizeof(int);
+    }
+
+    /* ??? */
+    if (ios->ioproc)
+    {
+        /* Allocate memory to hold array of tasks that have recieved
+         * data ??? */
+        if (!(recv_buf = calloc(ntasks, sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Initialize arrays that keep track of receives. */
+        for (i = 0; i < ntasks; i++)
+        {
+            recv_counts[i] = 1;
+            recv_displs[i] = i * sizeof(int);
+        }
+    }
+
+    /* Share the iodesc->scount from each compute task to all io tasks. */
+    ierr = pio_swapm(iodesc->scount, send_counts, send_displs, sr_types,
+                     recv_buf, recv_counts, recv_displs, sr_types,
+                     mycomm,
+                     iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs,
+                     iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend,
+                     iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req);
+
+    /* ??? */
+    nrecvs = 0;
+    if (ios->ioproc)
+    {
+        for (i = 0; i < ntasks; i++)
+            if (recv_buf[i] != 0)
+                nrecvs++;
+
+        /* Get memory to hold the count of data receives. */
+        if (!(iodesc->rcount = malloc(max(1, nrecvs) * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Get memory to hold the list of task data was from. */
+        if (!(iodesc->rfrom = malloc(max(1, nrecvs) * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        for (i = 0; i < max(1, nrecvs); i++)
+        {
+            iodesc->rcount[i] = 0;
+            iodesc->rfrom[i] = 0;
+        }
+
+        nrecvs = 0;
+        for (i = 0; i < ntasks; i++)
+        {
+            if (recv_buf[i] != 0)
+            {
+                iodesc->rcount[nrecvs] = recv_buf[i];
+                iodesc->rfrom[nrecvs] = i;
+                nrecvs++;
+            }
+        }
+        free(recv_buf);
+    }
+
+    /* ??? */
+    iodesc->nrecvs = nrecvs;
+    if (iodesc->sindex == NULL && iodesc->ndof > 0)
+        if (!(iodesc->sindex = malloc(iodesc->ndof * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    int tempcount[numiotasks];
+    int spos[numiotasks];
+
+    /* ??? */
+    spos[0] = 0;
+    tempcount[0] = 0;
+    for (i = 1; i < numiotasks; i++)
+    {
+        spos[i] = spos[i - 1] + iodesc->scount[i - 1];
+        tempcount[i] = 0;
+    }
+
+    for (i = 0; i < maplen; i++)
+    {
+        iorank = dest_ioproc[i];
+        ioindex = dest_ioindex[i];
+        if (iorank > -1)
+        {
+            /* this should be moved to create_box */
+            if (iodesc->rearranger == PIO_REARR_BOX)
+                iodesc->sindex[spos[iorank] + tempcount[iorank]] = i;
+
+            s2rindex[spos[iorank] + tempcount[iorank]] = ioindex;
+            (tempcount[iorank])++;
+        }
+    }
+
+    /* Initialize arrays to zeros. */
+    for (i = 0; i < ntasks; i++)
+    {
+        send_counts[i] = 0;
+        send_displs[i] = 0;
+        recv_counts[i] = 0;
+        recv_displs[i] = 0;
+    }
+
+    /* Find the size of the offset type. */
+    if ((mpierr = MPI_Type_size(MPI_OFFSET, &tsize)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    for (i = 0; i < ntasks; i++)
+        sr_types[i] = MPI_OFFSET;
+
+    for (i = 0; i < numiotasks; i++)
+    {
+        if (iodesc->rearranger == PIO_REARR_BOX)
+            io_comprank = ios->ioranks[i];
+        else
+            io_comprank = 0;
+
+        send_counts[io_comprank] = iodesc->scount[i];
+        if (send_counts[io_comprank] > 0)
+            send_displs[io_comprank] = spos[i] * tsize;
+    }
+
+    if (ios->ioproc)
+    {
+        int totalrecv = 0;
+        for (i = 0; i < nrecvs; i++)
+        {
+            recv_counts[iodesc->rfrom[i]] = iodesc->rcount[i];
+            totalrecv+=iodesc->rcount[i];
+        }
+        recv_displs[0] = 0;
+        for (i = 1; i < nrecvs; i++)
+            recv_displs[iodesc->rfrom[i]] = recv_displs[iodesc->rfrom[i-1]] + iodesc->rcount[i-1] * tsize;
+
+
+        if (totalrecv > 0)
+        {
+            totalrecv = iodesc->llen;  /* can reduce memory usage here */
+            if (!(iodesc->rindex = calloc(totalrecv, sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+    }
+
+    /* s2rindex is the list of indeces on each compute task */
+    /*
+      printf("%d s2rindex: ", ios->comp_rank);
+      for (i = 0;indof;i++)
+      printf("%ld ",s2rindex[i]);
+      printf("\n");
+    */
+    ierr = pio_swapm(s2rindex, send_counts, send_displs, sr_types, iodesc->rindex,
+                     recv_counts, recv_displs, sr_types, mycomm,
+                     iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs,
+                     iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend,
+                     iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req);
+
+    /*  rindex is an array of the indices of the data to be sent from
+        this io task to each compute task. */
+    /*
+      if (ios->ioproc){
+      printf("%d rindex: ",ios->io_rank);
+      for (int j = 0;jllen;j++)
+      printf(" %ld ",iodesc->rindex[j]);
+      printf("\n");
+      }
+    */
+    return ierr;
+}
+
+/**
+ * Moves data from compute tasks to IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t struct
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param sbuf send buffer.
+ * @param rbuf receive buffer.
+ * @param nvars number of variables.
+ * @returns 0 on success, error code otherwise.
+ */
+int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
+                      void *rbuf, int nvars)
+{
+    int ntasks;
+    int niotasks;
+    int *scount = iodesc->scount;
+    int i, tsize;
+    int *sendcounts;
+    int *recvcounts;
+    int *sdispls;
+    int *rdispls;
+    MPI_Datatype *sendtypes;
+    MPI_Datatype *recvtypes;
+    MPI_Comm mycomm;
+    int mpierr; /* Return code from MPI calls. */
+    int ret;
+
+#ifdef TIMING
+    GPTLstart("PIO:rearrange_comp2io");
+#endif
+
+    /* Caller must provide these. */
+    pioassert(iodesc && nvars > 0, "invalid input", __FILE__, __LINE__);
+
+    LOG((2, "rearrange_comp2io nvars = %d iodesc->rearranger = %d", nvars,
+         iodesc->rearranger));
+
+    if (iodesc->rearranger == PIO_REARR_BOX)
+    {
+        mycomm = ios->union_comm;
+        niotasks = ios->num_iotasks;
+    }
+    else
+    {
+        mycomm = iodesc->subset_comm;
+        niotasks = 1;
+    }
+
+    /* Get the number of tasks. */
+    if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    /* Get the size of the MPI type. */
+    if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    LOG((3, "ntasks = %d tsize = %d", ntasks, tsize));
+
+    /* Define the MPI data types that will be used for this
+     * io_desc_t. */
+    if ((ret = define_iodesc_datatypes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Allocate arrays needed by the pio_swapm() function. */
+    if (!(sendcounts = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(recvcounts = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(sdispls = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(rdispls = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(sendtypes = malloc(ntasks * sizeof(MPI_Datatype))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(recvtypes = malloc(ntasks * sizeof(MPI_Datatype))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Initialize arrays. */
+    for (i = 0; i < ntasks; i++)
+    {
+        recvtypes[i] = PIO_DATATYPE_NULL;
+        sendtypes[i] =  PIO_DATATYPE_NULL;
+    }
+
+    /* If this io proc will exchange data with compute tasks create a
+     * MPI DataType for that exchange. */
+    if (ios->ioproc && iodesc->nrecvs > 0)
+    {
+        for (i = 0; i < iodesc->nrecvs; i++)
+        {
+            if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
+            {
+                if (iodesc->rearranger == PIO_REARR_SUBSET)
+                {
+                    recvcounts[ i ] = 1;
+
+                    /*  The stride here is the length of the collected array (llen) */
+
+                    if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint) iodesc->llen * tsize, iodesc->rtype[i],
+                                                   recvtypes + i)))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                    if (recvtypes[i] == PIO_DATATYPE_NULL)
+                        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+                    if ((mpierr = MPI_Type_commit(recvtypes + i)))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                    /*recvtypes[ i ] = iodesc->rtype[i]; */
+                }
+                else
+                {
+                    recvcounts[iodesc->rfrom[i]] = 1;
+
+                    if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * tsize, iodesc->rtype[i],
+                                                   recvtypes + iodesc->rfrom[i])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                    if (recvtypes[iodesc->rfrom[i]] == PIO_DATATYPE_NULL)
+                        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+                    if ((mpierr = MPI_Type_commit(recvtypes+iodesc->rfrom[i])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                    /* recvtypes[ iodesc->rfrom[i] ] = iodesc->rtype[i]; */
+                    rdispls[iodesc->rfrom[i]] = 0;
+                }
+            }
+        }
+    }
+
+    /* On compute tasks loop over iotasks and create a data type for
+     * each exchange.  */
+    for (i = 0; i < niotasks; i++)
+    {
+        int io_comprank = ios->ioranks[i];
+        if (iodesc->rearranger == PIO_REARR_SUBSET)
+            io_comprank=0;
+
+        if (scount[i] > 0 && sbuf != NULL)
+        {
+            sendcounts[io_comprank] = 1;
+            if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * tsize, iodesc->stype[i],
+                                           sendtypes + io_comprank)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            if (sendtypes[io_comprank] == PIO_DATATYPE_NULL)
+                return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+            if ((mpierr = MPI_Type_commit(sendtypes + io_comprank)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+        else
+        {
+            sendcounts[io_comprank]=0;
+        }
+    }
+    /* Data in sbuf on the compute nodes is sent to rbuf on the ionodes */
+    pio_swapm(sbuf, sendcounts, sdispls, sendtypes,
+              rbuf, recvcounts, rdispls, recvtypes, mycomm,
+              iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs,
+              iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend,
+              iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req);
+
+    /* Free the MPI types. */
+    for (i = 0; i < ntasks; i++)
+    {
+        if (sendtypes[i] != PIO_DATATYPE_NULL)
+            if ((mpierr = MPI_Type_free(sendtypes + i)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        if (recvtypes[i] != PIO_DATATYPE_NULL)
+            if ((mpierr = MPI_Type_free(recvtypes + i)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Free memory. */
+    free(sendcounts);
+    free(recvcounts);
+    free(sdispls);
+    free(rdispls);
+    free(sendtypes);
+    free(recvtypes);
+
+#ifdef TIMING
+    GPTLstop("PIO:rearrange_comp2io");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Moves data from IO tasks to compute tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param sbuf send buffer.
+ * @param rbuf receive buffer.
+ * @returns 0 on success, error code otherwise.
+ */
+int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
+                      void *rbuf)
+{
+    MPI_Comm mycomm;
+    int ntasks;
+    int niotasks;
+    int *scount = iodesc->scount;
+    int *sendcounts;
+    int *recvcounts;
+    int *sdispls;
+    int *rdispls;
+    MPI_Datatype *sendtypes;
+    MPI_Datatype *recvtypes;
+    int i;
+    int mpierr; /* Return code from MPI calls. */
+    int ret;
+
+    assert(iodesc);
+
+#ifdef TIMING
+    GPTLstart("PIO:rearrange_io2comp");
+#endif
+
+    /* Different rearrangers use different communicators and number of
+     * IO tasks. */
+    if (iodesc->rearranger == PIO_REARR_BOX)
+    {
+        mycomm = ios->union_comm;
+        niotasks = ios->num_iotasks;
+    }
+    else
+    {
+        mycomm = iodesc->subset_comm;
+        niotasks=1;
+    }
+
+    /* Get the size of this communicator. */
+    if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Define the MPI data types that will be used for this
+     * io_desc_t. */
+    if ((ret = define_iodesc_datatypes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Allocate arrays needed by the pio_swapm() function. */
+    if (!(sendcounts = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(recvcounts = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(sdispls = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(rdispls = calloc(ntasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(sendtypes = malloc(ntasks * sizeof(MPI_Datatype))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(recvtypes = malloc(ntasks * sizeof(MPI_Datatype))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Initialize arrays. */
+    for (i = 0; i < ntasks; i++)
+    {
+        sendtypes[i] = PIO_DATATYPE_NULL;
+        recvtypes[i] = PIO_DATATYPE_NULL;
+    }
+
+    /* In IO tasks ??? */
+    if (ios->ioproc)
+        for (i = 0; i < iodesc->nrecvs; i++)
+            if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
+            {
+                if (iodesc->rearranger == PIO_REARR_SUBSET)
+                {
+                    if (sbuf)
+                    {
+                        sendcounts[i] = 1;
+                        sendtypes[i] = iodesc->rtype[i];
+                    }
+                }
+                else
+                {
+                    sendcounts[iodesc->rfrom[i]] = 1;
+                    sendtypes[iodesc->rfrom[i]] = iodesc->rtype[i];
+                }
+            }
+
+    /* In the box rearranger each comp task may communicate with
+     * multiple IO tasks here we are setting the count and data type
+     * of the communication of a given compute task with each io
+     * task. */
+    for (i = 0; i < niotasks; i++)
+    {
+        int io_comprank = ios->ioranks[i];
+        if (iodesc->rearranger == PIO_REARR_SUBSET)
+            io_comprank = 0;
+
+        if (scount[i] > 0 && iodesc->stype[i] != PIO_DATATYPE_NULL)
+        {
+            recvcounts[io_comprank] = 1;
+            recvtypes[io_comprank] = iodesc->stype[i];
+        }
+    }
+
+    /* Data in sbuf on the ionodes is sent to rbuf on the compute nodes */
+    pio_swapm(sbuf, sendcounts, sdispls, sendtypes,
+              rbuf, recvcounts, rdispls, recvtypes, mycomm,
+              iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs,
+              iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend,
+              iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req);
+
+    /* Release memory. */
+    free(sendcounts);
+    free(recvcounts);
+    free(sdispls);
+    free(rdispls);
+    free(sendtypes);
+    free(recvtypes);
+
+#ifdef TIMING
+    GPTLstop("PIO:rearrange_io2comp");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Determine fill value.
+ *
+ * @param ios pointer to the iosystem_desc_t struct
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param gsize pointer to an array of sizes
+ * @param compmap
+ * @returns 0 on success, error code otherwise.
+ */
+int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize,
+                   const PIO_Offset *compmap)
+{
+    PIO_Offset totalllen = 0;
+    PIO_Offset totalgridsize = 1;
+    int i;
+    int mpierr; /* Return code from MPI calls. */
+
+    assert(iodesc);
+
+    for (i = 0; i < iodesc->ndims; i++)
+        totalgridsize *= gsize[i];
+
+    if (iodesc->rearranger == PIO_REARR_SUBSET)
+        totalllen=iodesc->llen;
+    else
+        for (i = 0; indof; i++)
+            if (compmap[i] > 0)
+                totalllen++;
+
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totalllen, 1, PIO_OFFSET, MPI_SUM,
+                                ios->union_comm)))
+        check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    /* If the total size of the data provided to be written is < the
+     * total data size then we need fill values. */
+    if (totalllen < totalgridsize)
+        iodesc->needsfill = true;
+    else
+        iodesc->needsfill = false;
+
+    /*  TURN OFF FILL for timing test
+        iodesc->needsfill=false; */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Prints the IO desc information to stdout.
+ *
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+void iodesc_dump(io_desc_t *iodesc)
+{
+    assert(iodesc);
+
+    printf("ioid= %d\n", iodesc->ioid);
+/*    printf("async_id= %d\n",iodesc->async_id);*/
+    printf("nrecvs= %d\n", iodesc->nrecvs);
+    printf("ndof= %d\n", iodesc->ndof);
+    printf("ndims= %d\n", iodesc->ndims);
+    printf("num_aiotasks= %d\n", iodesc->num_aiotasks);
+    printf("rearranger= %d\n", iodesc->rearranger);
+    printf("maxregions= %d\n", iodesc->maxregions);
+    printf("needsfill= %d\n", (int)iodesc->needsfill);
+
+    printf("llen= %lld\n", iodesc->llen);
+    printf("maxiobuflen= %d\n", iodesc->maxiobuflen);
+
+    printf("rindex= ");
+    for (int j = 0; j < iodesc->llen; j++)
+        printf(" %lld ", iodesc->rindex[j]);
+    printf("\n");
+}
+
+/**
+ * The box rearranger computes a mapping between IO tasks and compute
+ * tasks such that the data on io tasks can be written with a single
+ * call to the underlying netcdf library.  This may involve an all to
+ * all rearrangement in the mapping, but should minimize data movement
+ * in lower level libraries
+ *
+ * @param ios pointer to the iosystem_desc_t struct
+ * @param maplen the length of the map
+ * @param compmap
+ * @param gsize pointer to an array of sizes
+ * @param ndims the number of dimensions
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap,
+                         const int *gsize, int ndims, io_desc_t *iodesc)
+{
+    int nprocs = ios->num_comptasks;
+    int nioprocs = ios->num_iotasks;
+    PIO_Offset gstride[ndims];
+    PIO_Offset start[ndims], count[ndims];
+    int  tsize, i, j, k;
+    int dest_ioproc[maplen];
+    PIO_Offset dest_ioindex[maplen];
+    int sndlths[nprocs];
+    int sdispls[nprocs];
+    int recvlths[nprocs];
+    int rdispls[nprocs];
+    MPI_Datatype dtypes[nprocs];
+    PIO_Offset iomaplen[nioprocs];
+    int mpierr; /* Return code from MPI functions. */
+    int ret;
+
+    assert(iodesc);
+
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    iodesc->ndof = maplen;
+    gstride[ndims-1] = 1;
+    for (int i= ndims - 2; i >= 0; i--)
+        gstride[i] = gstride[i + 1] * gsize[i + 1];
+
+    if ((mpierr = MPI_Type_size(MPI_OFFSET, &tsize)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    for (i = 0; i < maplen; i++)
+    {
+        dest_ioproc[i] = -1;
+        dest_ioindex[i] = -1;
+    }
+    for (i = 0; i < nprocs; i++)
+    {
+        sndlths[i] = 0;
+        sdispls[i] = 0;
+        recvlths[i] = 0;
+        rdispls[i] = 0;
+        dtypes[i] = MPI_OFFSET;
+    }
+    iodesc->llen = 0;
+    if (ios->ioproc)
+    {
+        for (i = 0; i < nprocs; i++)
+            sndlths[i] = 1;
+
+        /* llen here is the number that will be read on each io task */
+        iodesc->llen = 1;
+        for (i = 0; i < ndims; i++)
+            iodesc->llen *= iodesc->firstregion->count[i];
+    }
+    if ((ret = determine_fill(ios, iodesc, gsize, compmap)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /*
+      if (ios->ioproc){
+      for (i = 0; ifirstregion->start[i],iodesc->firstregion->count[i]);
+      }
+      printf("\n%s %d\n",__FILE__,__LINE__);
+      }
+    */
+
+    for (i = 0; i < nioprocs; i++)
+    {
+        int io_comprank = ios->ioranks[i];
+        recvlths[io_comprank] = 1;
+        rdispls[io_comprank] = i * tsize;
+    }
+
+    /*  The length of each iomap
+        iomaplen = calloc(nioprocs, sizeof(PIO_Offset)); */
+    pio_swapm(&(iodesc->llen), sndlths, sdispls, dtypes,
+              iomaplen, recvlths, rdispls, dtypes,
+              ios->union_comm,
+              iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs,
+              iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend,
+              iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req);
+
+    for (i = 0; i < nioprocs; i++)
+    {
+
+        if (iomaplen[i] > 0)
+        {
+            int io_comprank = ios->ioranks[i];
+            for (j = 0; j < nprocs; j++)
+            {
+                sndlths[j] = 0;
+                sdispls[j] = 0;
+                rdispls[j] = 0;
+                recvlths[j] = 0;
+                if (ios->union_rank == io_comprank)
+                    sndlths[j] = ndims;
+            }
+            recvlths[io_comprank] = ndims;
+
+            /* The count from iotask i is sent to all compute tasks */
+            pio_swapm(iodesc->firstregion->count,  sndlths, sdispls, dtypes,
+                      count, recvlths, rdispls, dtypes,
+                      ios->union_comm,
+                      iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs,
+                      iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend,
+                      iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req);
+
+            /* The start from iotask i is sent to all compute tasks. */
+            pio_swapm(iodesc->firstregion->start,  sndlths, sdispls, dtypes,
+                      start, recvlths, rdispls, dtypes,
+                      ios->union_comm,
+                      iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs,
+                      iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend,
+                      iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req);
+
+            for (k = 0; k < maplen; k++)
+            {
+                PIO_Offset gcoord[ndims], lcoord[ndims];
+                bool found = true;
+                /* The compmap array is 1 based but calculations are 0 based */
+                idx_to_dim_list(ndims, gsize, compmap[k] - 1, gcoord);
+
+                for (j = 0; j < ndims; j++)
+                {
+                    if (gcoord[j] >= start[j] && gcoord[j] < start[j] + count[j])
+                    {
+                        lcoord[j] = gcoord[j] - start[j];
+                    }
+                    else
+                    {
+                        found = false;
+                        break;
+                    }
+                }
+                if (found)
+                {
+                    dest_ioindex[k] = coord_to_lindex(ndims, lcoord, count);
+                    dest_ioproc[k] = i;
+                }
+            }
+        }
+    }
+
+    /* Check that a destination is found for each compmap entry. */
+    for (k = 0; k < maplen; k++)
+        if (dest_ioproc[k] < 0 && compmap[k] > 0)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    compute_counts(ios, iodesc, maplen, dest_ioproc, dest_ioindex, ios->union_comm);
+
+    if (ios->ioproc)
+        if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Using maxiobuflen compute the maximum number of vars of this type that the io
+       task buffer can handle. */
+    if ((ret = compute_maxaggregate_bytes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+#ifdef DEBUG
+    iodesc_dump(iodesc);
+#endif
+    return PIO_NOERR;
+}
+
+/**
+ * Compare offsets is used by the sort in the subset rearrange. This
+ * function is passed to qsort.
+ *
+ * @param a
+ * @param b
+ * @returns 0 if offsets are the same.
+ */
+int compare_offsets(const void *a, const void *b)
+{
+    mapsort *x = (mapsort *)a;
+    mapsort *y = (mapsort *)b;
+    if (!x || !y)
+        return 0;
+    return (int)(x->iomap - y->iomap);
+}
+
+/**
+ * Each region is a block of output which can be represented in a
+ * single call to the underlying netcdf library.  This can be as small
+ * as a single data point, but we hope we've aggragated better than
+ * that.
+ *
+ * @param ndims the number of dimensions
+ * @param gdims pointer to an array of dimension ids
+ * @param maplen the length of the map
+ * @param map
+ * @param maxregions
+ * @param firstregion pointer to the first region.
+ * @returns 0 on success, error code otherwise.
+ */
+void get_start_and_count_regions(int ndims, const int *gdims, int maplen, const PIO_Offset *map,
+                                 int *maxregions, io_region *firstregion)
+{
+    int i;
+    int nmaplen;
+    int regionlen;
+    io_region *region;
+
+    assert(maxregions);
+    assert(firstregion);
+
+    nmaplen = 0;
+    region = firstregion;
+    if (map)
+    {
+        while(map[nmaplen++] <= 0);
+        nmaplen--;
+    }
+    region->loffset = nmaplen;
+
+    *maxregions = 1;
+
+    while(nmaplen < maplen)
+    {
+        /* Here we find the largest region from the current offset
+           into the iomap regionlen is the size of that region and we
+           step to that point in the map array until we reach the
+           end */
+        for (i = 0; i < ndims; i++)
+            region->count[i] = 1;
+
+        regionlen = find_region(ndims, gdims, maplen-nmaplen,
+                                map+nmaplen, region->start, region->count);
+
+        pioassert(region->start[0] >= 0, "failed to find region", __FILE__, __LINE__);
+
+        nmaplen = nmaplen + regionlen;
+
+        if (region->next == NULL && nmaplen < maplen)
+        {
+            region->next = alloc_region(ndims);
+            /* The offset into the local array buffer is the sum of
+             * the sizes of all of the previous regions (loffset) */
+            region = region->next;
+            region->loffset = nmaplen;
+
+            /* The calls to the io library are collective and so we
+               must have the same number of regions on each io task
+               maxregions will be the total number of regions on this
+               task. */
+            (*maxregions)++;
+        }
+    }
+}
+
+/**
+ * The subset rearranger needs a mapping from compute tasks to IO
+ * task, the only requirement is that each compute task map to one and
+ * only one IO task.  This mapping groups by mpi task id others are
+ * possible and may be better for certain decompositions
+ *
+ * @param ios pointer to the iosystem_desc_t struct
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int taskratio = ios->num_comptasks/ios->num_iotasks;
+    int color;
+    int key;
+    int mpierr; /* Return value from MPI functions. */
+
+    assert(ios && iodesc);
+
+    /* Create a new comm for each subset group with the io task in
+       rank 0 and only 1 io task per group */
+    if (ios->ioproc)
+    {
+        key = 0;
+        color= ios->io_rank;
+    }
+    else
+    {
+        key = max(1, ios->comp_rank % taskratio + 1);
+        color = min(ios->num_iotasks - 1, ios->comp_rank / taskratio);
+    }
+
+    if ((mpierr = MPI_Comm_split(ios->comp_comm, color, key, &iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * The subset rearranger computes a mapping between IO tasks and
+ * compute tasks such that each compute task communicates with one and
+ * only one IO task.
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param maplen the length of the map.
+ * @param compmap
+ * @param gsize pointer to an array of sizes.
+ * @param ndims the number of dimensions.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap,
+                            const int *gsize, int ndims, io_desc_t *iodesc)
+{
+    int i, j;
+    PIO_Offset *iomap = NULL;
+    int ierr = PIO_NOERR;
+    mapsort *map = NULL;
+    PIO_Offset totalgridsize;
+    PIO_Offset *srcindex = NULL;
+    PIO_Offset *myfillgrid = NULL;
+    int maxregions;
+    int rank, ntasks, rcnt;
+    int mpierr; /* Return call from MPI function calls. */
+    int ret;
+
+    pioassert(iodesc, "iodesc must be provided", __FILE__, __LINE__);
+
+    LOG((2, "subset_rearrange_create maplen = %d ndims = %d", maplen, ndims));
+
+    /* subset partitions each have exactly 1 io task which is task 0
+     * of that subset_comm */
+    /* TODO: introduce a mechanism for users to define partitions */
+    if ((ret = default_subset_partition(ios, iodesc)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    iodesc->rearranger = PIO_REARR_SUBSET;
+
+    /* Get size of this subset communicator, and rank in it. */
+    if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Check rank for correctness. */
+    if (ios->ioproc)
+        pioassert(rank == 0, "Bad io rank in subset create", __FILE__, __LINE__);
+    else
+        pioassert(rank > 0 && rank < ntasks, "Bad comp rank in subset create",
+                  __FILE__, __LINE__);
+
+    rcnt = 0;
+    iodesc->ndof = maplen;
+    if (ios->ioproc)
+    {
+        /* Allocate space to hold count of data to be received in pio_swapm(). */
+        if (!(iodesc->rcount = malloc(ntasks * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        rcnt = 1;
+    }
+
+    /* Allocate space to hold count of data to be sent in pio_swapm(). */
+    if (!(iodesc->scount = malloc(sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    iodesc->scount[0] = 0;
+    totalgridsize = 1;
+    for (i = 0; i < ndims; i++)
+        totalgridsize *= gsize[i];
+
+    for (i = 0; i < maplen; i++)
+    {
+        /*  turns out this can be allowed in some cases
+            pioassert(compmap[i]>=0 && compmap[i]<=totalgridsize, "Compmap value out of bounds",
+            __FILE__,__LINE__); */
+        if (compmap[i] > 0)
+            (iodesc->scount[0])++;
+    }
+
+    if (iodesc->scount[0] > 0)
+        if (!(iodesc->sindex = calloc(iodesc->scount[0], sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    j = 0;
+    for (i = 0; i < maplen; i++)
+        if (compmap[i] > 0)
+            iodesc->sindex[j++] = i;
+
+    /* Pass the reduced maplen (without holes) from each compute task to its associated IO task
+       printf("%s %d %ld\n",__FILE__,__LINE__,iodesc->scount); */
+    if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt, MPI_INT,
+                             0, iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    iodesc->llen = 0;
+
+    int rdispls[ntasks];
+    int recvlths[ntasks];
+
+    if (ios->ioproc)
+    {
+        for (i = 0;i < ntasks; i++)
+        {
+            iodesc->llen += iodesc->rcount[i];
+            rdispls[i] = 0;
+            recvlths[i] = iodesc->rcount[i];
+            if (i > 0)
+                rdispls[i] = rdispls[i - 1] + iodesc->rcount[i - 1];
+        }
+
+        if (iodesc->llen > 0)
+        {
+            if (!(srcindex = calloc(iodesc->llen, sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+            for (i = 0; i < iodesc->llen; i++)
+                srcindex[i] = 0;
+        }
+    }
+    else
+    {
+        for (i = 0; i < ntasks; i++)
+        {
+            recvlths[i] = 0;
+            rdispls[i] = 0;
+        }
+    }
+    if ((ret = determine_fill(ios, iodesc, gsize, compmap)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Pass the sindex from each compute task to its associated IO task. */
+    if ((mpierr = MPI_Gatherv(iodesc->sindex, iodesc->scount[0], PIO_OFFSET,
+                              srcindex, recvlths, rdispls, PIO_OFFSET, 0,
+                              iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    if (ios->ioproc && iodesc->llen>0)
+    {
+        if (!(map = calloc(iodesc->llen, sizeof(mapsort))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(iomap = calloc(iodesc->llen, sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    }
+
+    /* Now pass the compmap, skipping the holes. */
+    PIO_Offset *shrtmap;
+    if (maplen>iodesc->scount[0] && iodesc->scount[0] > 0)
+    {
+        if (!(shrtmap = calloc(iodesc->scount[0], sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        j = 0;
+        for (i = 0; i < maplen; i++)
+            if (compmap[i] > 0)
+                shrtmap[j++] = compmap[i];
+    }
+    else
+    {
+        shrtmap = compmap;
+    }
+
+    if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvlths, rdispls,
+                              PIO_OFFSET, 0, iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    if (shrtmap != compmap)
+        free(shrtmap);
+
+    if (ios->ioproc && iodesc->llen > 0)
+    {
+        int pos = 0;
+        int k = 0;
+        mapsort *mptr;
+        for (i = 0; i < ntasks; i++)
+        {
+            for (j = 0; j < iodesc->rcount[i]; j++)
+            {
+                mptr = map + k;
+                mptr->rfrom = i;
+                mptr->soffset = srcindex[pos + j];
+                mptr->iomap = iomap[pos + j];
+                k++;
+            }
+            pos += iodesc->rcount[i];
+        }
+        /* sort the mapping, this will transpose the data into IO order */
+        qsort(map, iodesc->llen, sizeof(mapsort), compare_offsets);
+
+        if (!(iodesc->rindex = calloc(1, iodesc->llen * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(iodesc->rfrom = calloc(1, iodesc->llen * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    }
+
+    int cnt[ntasks];
+    for (i = 0; i < ntasks; i++)
+    {
+        cnt[i] = rdispls[i];
+
+        /* offsets to swapm are in bytes */
+        /*    rdispls[i]*=pio_offset_size; */
+    }
+
+    mapsort *mptr;
+    for (i = 0; i < iodesc->llen; i++)
+    {
+        mptr = map+i;
+        iodesc->rfrom[i] = mptr->rfrom;
+        iodesc->rindex[i] = i;
+        iomap[i] = mptr->iomap;
+        srcindex[(cnt[iodesc->rfrom[i]])++] = mptr->soffset;
+    }
+
+    if (ios->ioproc && iodesc->needsfill)
+    {
+        /* we need the list of offsets which are not in the union of iomap */
+        PIO_Offset thisgridsize[ios->num_iotasks];
+        PIO_Offset thisgridmin[ios->num_iotasks], thisgridmax[ios->num_iotasks];
+        int nio;
+        PIO_Offset *myusegrid = NULL;
+        int gcnt[ios->num_iotasks];
+        int displs[ios->num_iotasks];
+
+        thisgridmin[0] = 1;
+        thisgridsize[0] =  totalgridsize / ios->num_iotasks;
+        thisgridmax[0] = thisgridsize[0];
+        int xtra = totalgridsize - thisgridsize[0] * ios->num_iotasks;
+
+        for (nio = 0; nio < ios->num_iotasks; nio++)
+        {
+            int cnt = 0;
+            int imin = 0;
+            if (nio > 0)
+            {
+                thisgridsize[nio] =  totalgridsize / ios->num_iotasks;
+                if (nio >= ios->num_iotasks - xtra)
+                    thisgridsize[nio]++;
+                thisgridmin[nio] = thisgridmax[nio - 1] + 1;
+                thisgridmax[nio]= thisgridmin[nio] + thisgridsize[nio] - 1;
+            }
+            for (int i = 0; i < iodesc->llen; i++)
+            {
+                if (iomap[i] >= thisgridmin[nio] && iomap[i] <= thisgridmax[nio])
+                {
+                    cnt++;
+                    if (cnt == 1)
+                        imin = i;
+                }
+            }
+
+            if ((mpierr = MPI_Gather(&cnt, 1, MPI_INT, gcnt, 1, MPI_INT, nio, ios->io_comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            if (nio == ios->io_rank)
+            {
+                displs[0] = 0;
+                for (i = 1; i < ios->num_iotasks; i++)
+                    displs[i] = displs[i - 1] + gcnt[i - 1];
+
+                /* Allocate storage for the grid. */
+                if (!(myusegrid = malloc(thisgridsize[nio] * sizeof(PIO_Offset))))
+                    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+                /* Initialize the grid to all -1. */
+                for (i = 0; i < thisgridsize[nio]; i++)
+                    myusegrid[i] = -1;
+            }
+
+            if ((mpierr = MPI_Gatherv((iomap + imin), cnt, PIO_OFFSET, myusegrid, gcnt,
+                                      displs, PIO_OFFSET, nio, ios->io_comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+
+        PIO_Offset grid[thisgridsize[ios->io_rank]];
+        for (i = 0; i < thisgridsize[ios->io_rank]; i++)
+            grid[i] = 0;
+
+        int cnt = 0;
+        for (i = 0; i < thisgridsize[ios->io_rank]; i++)
+        {
+            int j = myusegrid[i] - thisgridmin[ios->io_rank];
+            pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index", __FILE__, __LINE__);
+            if (j >= 0)
+            {
+                grid[j] = 1;
+                cnt++;
+            }
+        }
+        if (myusegrid)
+            free(myusegrid);
+
+        iodesc->holegridsize=thisgridsize[ios->io_rank] - cnt;
+        if (iodesc->holegridsize > 0)
+        {
+            /* Allocate space for the fillgrid. */
+            if (!(myfillgrid = malloc(iodesc->holegridsize * sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+
+        /* Initialize the fillgrid. */
+        for (i = 0; i < iodesc->holegridsize; i++)
+            myfillgrid[i] = -1;
+
+        j = 0;
+        for (i = 0; i < thisgridsize[ios->io_rank]; i++)
+        {
+            if (grid[i] == 0)
+            {
+                if (myfillgrid[j] == -1)
+                    myfillgrid[j++]=thisgridmin[ios->io_rank] + i;
+                else
+                    return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+            }
+        }
+        maxregions = 0;
+        iodesc->maxfillregions = 0;
+        if (myfillgrid)
+        {
+            iodesc->fillregion = alloc_region(iodesc->ndims);
+            get_start_and_count_regions(iodesc->ndims, gsize, iodesc->holegridsize, myfillgrid,
+                                        &iodesc->maxfillregions, iodesc->fillregion);
+            free(myfillgrid);
+            maxregions = iodesc->maxfillregions;
+        }
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        iodesc->maxfillregions = maxregions;
+
+	iodesc->maxholegridsize = iodesc->holegridsize;
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT, MPI_MAX, ios->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    if ((mpierr = MPI_Scatterv((void *)srcindex, recvlths, rdispls, PIO_OFFSET, (void *)iodesc->sindex,
+                               iodesc->scount[0],  PIO_OFFSET, 0, iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if (ios->ioproc)
+    {
+        iodesc->maxregions = 0;
+        get_start_and_count_regions(iodesc->ndims,gsize,iodesc->llen, iomap,&(iodesc->maxregions),
+                                    iodesc->firstregion);
+        maxregions = iodesc->maxregions;
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        iodesc->maxregions = maxregions;
+        if (iomap)
+            free(iomap);
+
+        if (map)
+            free(map);
+
+        if (srcindex)
+            free(srcindex);
+
+        compute_maxIObuffersize(ios->io_comm, iodesc);
+
+        iodesc->nrecvs = ntasks;
+#ifdef DEBUG
+        iodesc_dump(iodesc);
+#endif
+    }
+
+    /* Using maxiobuflen compute the maximum number of vars of this type that the io
+       task buffer can handle. */
+    if ((ret = compute_maxaggregate_bytes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return ierr;
+}
+
+/**
+ * Performance tuning rearranger.
+ *
+ * @param ios pointer to the iosystem description struct.
+ * @param iodesc pointer to the IO description struct.
+ * @returns 0 on success, error code otherwise.
+ */
+void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+#ifdef TIMING
+#ifdef PERFTUNE
+    double *wall, usr[2], sys[2];
+    void *cbuf, *ibuf;
+    int tsize;
+    int myrank;
+    int mpierr; /* Return code for MPI calls. */
+
+    assert(iodesc);
+
+    if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    cbuf = NULL;
+    ibuf = NULL;
+    if (iodesc->ndof > 0)
+        if (!(cbuf = bget(iodesc->ndof * tsize)))
+            piomemerror(ios, iodesc->ndof * tsize, __FILE__, __LINE__);
+
+    if (iodesc->llen > 0)
+        if (!(ibuf = bget(iodesc->llen * tsize)))
+            piomemerror(ios, iodesc->llen * tsize, __FILE__, __LINE__);
+
+    if (iodesc->rearranger == PIO_REARR_BOX)
+        mycomm = ios->union_comm;
+    else
+        mycomm = iodesc->subset_comm;
+
+    if ((mpierr = MPI_Comm_size(mycomm, &nprocs)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(mycomm, &myrank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    int log2 = log(nprocs) / log(2) + 1;
+    if (!(wall = bget(2 * 4 * log2 * sizeof(double))))
+        piomemerror(ios, 2 * 4 *log2 * sizeof(double), __FILE__, __LINE__);
+    double mintime;
+    int k = 0;
+
+    if ((mpierr = MPI_Barrier(mycomm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    GPTLstamp(&wall[0], &usr[0], &sys[0]);
+    rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1);
+    rearrange_io2comp(ios, iodesc, ibuf, cbuf);
+    GPTLstamp(&wall[1], &usr[1], &sys[1]);
+    mintime = wall[1]-wall[0];
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &mintime, 1, MPI_DOUBLE, MPI_MAX, mycomm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    handshake = iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs;
+    isend = iodesc->isend;
+    maxreqs = iodesc->max_requests;
+
+    for (int i = 0; i < 2; i++)
+    {
+        if (i == 0)
+            iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = false;
+        else
+            iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = true;
+
+        for (int j = 0; j < 2; j++)
+        {
+            if (j == 0)
+                iodesc->isend = false;
+            else
+                iodesc->isend = true;
+
+            iodesc->max_requests = 0;
+
+            for (nreqs = nprocs; nreqs >= 2; nreqs /= 2)
+            {
+                iodesc->max_requests = nreqs;
+                if ((mpierr = MPI_Barrier(mycomm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                GPTLstamp(wall, usr, sys);
+                rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1);
+                rearrange_io2comp(ios, iodesc, ibuf, cbuf);
+                GPTLstamp(wall+1, usr, sys);
+                wall[1] -= wall[0];
+                if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, wall + 1, 1, MPI_DOUBLE, MPI_MAX,
+                                            mycomm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                if (wall[1] < mintime * 0.95)
+                {
+                    handshake = iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs;
+                    isend = iodesc->isend;
+                    maxreqs = nreqs;
+                    mintime = wall[1];
+                }
+                else if (wall[1] > mintime * 1.05)
+                {
+                    exit;
+                }
+            }
+        }
+    }
+
+    iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = handshake;
+    iodesc->isend = isend;
+    iodesc->max_requests = maxreqs;
+
+    LOG((1, "spmd optimization: maxreqs: %d handshake:%d isend:%d mintime=%f\n",
+         maxreqs,handshake,isend,mintime));
+
+    /* Free memory. */
+    brel(wall);
+    brel(cbuf);
+    brel(ibuf);
+#endif
+#endif
+}
diff --git a/src/externals/pio2/src/clib/pio_spmd.c b/src/externals/pio2/src/clib/pio_spmd.c
new file mode 100644
index 00000000000..decf3f7e3bb
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_spmd.c
@@ -0,0 +1,509 @@
+/**
+ * @file
+ * Algorithms modeled after spmd_utils in the Community
+ * Atmosphere Model; C translation. This includes MPI_Gather,
+ * MPI_Gatherv, and MPI_Alltoallw with flow control options.
+ *
+ * @author Jim Edwards
+ * @date 2014
+ */
+#include 
+#include 
+#include 
+
+/**
+ * Returns the smallest power of 2 greater than
+ * or equal to i.
+ *
+ * @param i input number
+ * @returns the smallest power of 2 greater than
+ * or equal to i.
+ */
+int ceil2(int i)
+{
+    int p = 1;
+    while (p < i)
+        p *= 2;
+
+    return(p);
+}
+
+/**
+ * Given integers p and k between 0 and np-1,
+ * if (p+1)^k <= np-1 then return (p+1)^k else -1.
+ *
+ * @param np
+ * @param p integer between 0 and np - 1.
+ * @param k integer between 0 and np - 1.
+ * @returns (p + 1) ^ k else -1.
+ */
+int pair(int np, int p, int k)
+{
+    int q = (p + 1) ^ k ;
+    int pair = (q > np - 1) ? -1 : q;
+    return pair;
+}
+
+/**
+ * Provides the functionality of MPI_Alltoallw with flow control
+ * options. Generalized all-to-all communication allowing different
+ * datatypes, counts, and displacements for each partner
+ *
+ * @param sendbuf starting address of send buffer
+ * @param sendcounts integer array equal to the number of tasks in
+ * communicator comm (ntasks). It specifies the number of elements to
+ * send to each processor
+ * @param sdispls integer array (of length ntasks). Entry j
+ * specifies the displacement in bytes (relative to sendbuf) from
+ * which to take the outgoing data destined for process j.
+ * @param sendtypes array of datatypes (of length ntasks). Entry j
+ * specifies the type of data to send to process j.
+ * @param recvbuf address of receive buffer.
+ * @param recvcounts integer array (of length ntasks) specifying the
+ * number of elements that can be received from each processor.
+ * @param rdispls integer array (of length ntasks). Entry i
+ * specifies the displacement in bytes (relative to recvbuf) at which
+ * to place the incoming data from process i.
+ * @param recvtypes array of datatypes (of length ntasks). Entry i
+ * specifies the type of data received from process i.
+ * @param comm MPI communicator for the MPI_Alltoallw call.
+ * @param handshake if true, use handshaking.
+ * @param isend the isend bool indicates whether sends should be
+ * posted using mpi_irsend which can be faster than blocking
+ * sends. When flow control is used max_requests > 0 and the number of
+ * irecvs posted from a given task will not exceed this value. On some
+ * networks too many outstanding irecvs will cause a communications
+ * bottleneck.
+ * @param max_requests If 0, no flow control is used.
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes,
+              void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes,
+              MPI_Comm comm, bool handshake, bool isend, int max_requests)
+{
+    int ntasks;  /* Number of tasks in communicator comm. */
+    int my_rank; /* Rank of this task in comm. */
+    int tag;
+    int offset_t;
+    int steps;
+    int istep;
+    int rstep;
+    int p;
+    int maxreq;
+    int maxreqh;
+    int hs = 1; /* Used for handshaking. */
+    void *ptr;
+    MPI_Status status; /* Not actually used - replace with MPI_STATUSES_IGNORE. */
+    int mpierr;  /* Return code from MPI functions. */
+
+    LOG((2, "pio_swapm handshake = %d isend = %d max_requests = %d", handshake,
+         isend, max_requests));
+
+    /* Get my rank and size of communicator. */
+    if ((mpierr = MPI_Comm_size(comm, &ntasks)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &my_rank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "ntasks = %d my_rank = %d", ntasks, my_rank));
+
+    /* Now we know the size of these arrays. */
+    int swapids[ntasks];
+    MPI_Request rcvids[ntasks];
+    MPI_Request sndids[ntasks];
+    MPI_Request hs_rcvids[ntasks];
+
+    /* Print some debugging info, if logging is enabled. */
+#if PIO_ENABLE_LOGGING
+    {
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "sendcounts[%d] = %d", p, sendcounts[p]));
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "sdispls[%d] = %d", p, sdispls[p]));
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "sendtypes[%d] = %d", p, sendtypes[p]));
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "recvcounts[%d] = %d", p, recvcounts[p]));
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "rdispls[%d] = %d", p, rdispls[p]));
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "recvtypes[%d] = %d", p, recvtypes[p]));
+    }
+#endif /* PIO_ENABLE_LOGGING */
+
+    /* If max_requests == 0 no throttling is requested and the default
+     * mpi_alltoallw function is used. */
+    if (max_requests == 0)
+    {
+        /* Call the MPI alltoall without flow control. */
+        LOG((3, "Calling MPI_Alltoallw without flow control."));
+        if ((mpierr = MPI_Alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf,
+                                    recvcounts, rdispls, recvtypes, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return PIO_NOERR;
+    }
+
+    /* an index for communications tags */
+    offset_t = ntasks;
+
+    /* Send to self. */
+    if (sendcounts[my_rank] > 0)
+    {
+        void *sptr, *rptr;
+        tag = my_rank + offset_t;
+        sptr = (char *)sendbuf + sdispls[my_rank];
+        rptr = (char *)recvbuf + rdispls[my_rank];
+
+        /*
+          MPI_Type_get_extent(sendtypes[my_rank], &lb, &extent);
+          printf("%s %d %d %d\n",__FILE__,__LINE__,extent, lb);
+          MPI_Type_get_extent(recvtypes[my_rank], &lb, &extent);
+          printf("%s %d %d %d\n",__FILE__,__LINE__,extent, lb);
+        */
+
+#ifdef ONEWAY
+        /* If ONEWAY is true we will post mpi_sendrecv comms instead
+         * of irecv/send. */
+        if ((mpierr = MPI_Sendrecv(sptr, sendcounts[my_rank],sendtypes[my_rank],
+                                   my_rank, tag, rptr, recvcounts[my_rank], recvtypes[my_rank],
+                                   my_rank, tag, comm, &status)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+        if ((mpierr = MPI_Irecv(rptr, recvcounts[my_rank], recvtypes[my_rank],
+                                my_rank, tag, comm, rcvids)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Send(sptr, sendcounts[my_rank], sendtypes[my_rank],
+                               my_rank, tag, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        if ((mpierr = MPI_Wait(rcvids, &status)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif
+    }
+
+    LOG((2, "Done sending to self... sending to other procs"));
+
+    /* When send to self is complete there is nothing left to do if
+     * ntasks==1. */
+    if (ntasks == 1)
+        return PIO_NOERR;
+
+    for (int i = 0; i < ntasks; i++)
+    {
+        rcvids[i] = MPI_REQUEST_NULL;
+        swapids[i] = 0;
+    }
+
+    if (isend)
+        for (int i = 0; i < ntasks; i++)
+            sndids[i] = MPI_REQUEST_NULL;
+
+    if (handshake)
+        for (int i = 0; i < ntasks; i++)
+            hs_rcvids[i] = MPI_REQUEST_NULL;
+
+    steps = 0;
+    for (istep = 0; istep < ceil2(ntasks) - 1; istep++)
+    {
+        p = pair(ntasks, istep, my_rank);
+        if (p >= 0 && (sendcounts[p] > 0 || recvcounts[p] > 0))
+            swapids[steps++] = p;
+    }
+
+    LOG((3, "steps=%d", steps));
+
+    if (steps == 0)
+        return PIO_NOERR;
+
+    if (steps == 1)
+    {
+        maxreq = 1;
+        maxreqh = 1;
+    }
+    else
+    {
+        if (max_requests == PIO_REARR_COMM_UNLIMITED_PEND_REQ)
+        {
+            maxreq = steps;
+            maxreqh = steps;
+        }
+        else if (max_requests > 1 && max_requests < steps)
+        {
+            maxreq = max_requests;
+            maxreqh = maxreq / 2;
+        }
+        else if (max_requests == 1)
+        {
+            /* Note that steps >= 2 here */
+            maxreq = 2;
+            maxreqh = 1;
+        }
+        else
+        {
+            maxreq = steps;
+            maxreqh = steps;
+        }
+    }
+
+    LOG((2, "max_requests=%d, maxreq=%d, maxreqh=%d", max_requests, maxreq, maxreqh));
+
+    /* If handshaking is in use, do a nonblocking recieve to listen
+     * for it. */
+    if (handshake)
+    {
+        for (istep = 0; istep < maxreq; istep++)
+        {
+            p = swapids[istep];
+            if (sendcounts[p] > 0)
+            {
+                tag = my_rank + offset_t;
+                if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Post up to maxreq irecv's. */
+    for (istep = 0; istep < maxreq; istep++)
+    {
+        p = swapids[istep];
+        if (recvcounts[p] > 0)
+        {
+            tag = p + offset_t;
+            ptr = (char *)recvbuf + rdispls[p];
+
+            if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm,
+                                    rcvids + istep)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            if (handshake)
+                if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+    }
+
+    /* Tell the paired task that this tasks' has posted it's irecvs'. */
+    rstep = maxreq;
+    for (istep = 0; istep < steps; istep++)
+    {
+        p = swapids[istep];
+        if (sendcounts[p] > 0)
+        {
+            tag = my_rank + offset_t;
+            /* If handshake is enabled don't post sends until the
+             * receiving task has posted recvs. */
+            if (handshake)
+            {
+                if ((mpierr = MPI_Wait(hs_rcvids + istep, &status)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                hs_rcvids[istep] = MPI_REQUEST_NULL;
+            }
+            ptr = (char *)sendbuf + sdispls[p];
+
+            /* On some software stacks MPI_Irsend() is either not available, not
+             * a major issue anymore, or is buggy. With PIO1 we have found that
+             * although the code correctly posts receives before the irsends,
+             * on some systems (software stacks) the code hangs. However the
+             * code works fine with isends. The USE_MPI_ISEND_FOR_FC macro should be
+             * used to choose between mpi_irsends and mpi_isends - the default
+             * is still mpi_irsend
+             */
+            if (handshake && isend)
+            {
+#ifdef USE_MPI_ISEND_FOR_FC
+                if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
+                                        sndids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+                if ((mpierr = MPI_Irsend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
+                                         sndids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif
+            }
+            else if (isend)
+            {
+                if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
+                                         sndids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+            else
+            {
+                if ((mpierr = MPI_Send(ptr, sendcounts[p], sendtypes[p], p, tag, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+        }
+
+        /* We did comms in sets of size max_reqs, if istep > maxreqh-1
+         * then there is a remainder that must be handled. */
+        if (istep > maxreqh - 1)
+        {
+            p = istep - maxreqh;
+            if (rcvids[p] != MPI_REQUEST_NULL)
+            {
+                if ((mpierr = MPI_Wait(rcvids + p, &status)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                rcvids[p] = MPI_REQUEST_NULL;
+            }
+            if (rstep < steps)
+            {
+                p = swapids[rstep];
+                if (handshake && sendcounts[p] > 0)
+                {
+                    tag = my_rank + offset_t;
+                    if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids+rstep)))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                }
+                if (recvcounts[p] > 0)
+                {
+                    tag = p + offset_t;
+
+                    ptr = (char *)recvbuf + rdispls[p];
+                    if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm, rcvids + rstep)))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    if (handshake)
+                        if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm)))
+                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                }
+                rstep++;
+            }
+        }
+    }
+
+    /* If steps > 0 there could still be outstanding messages, wait for
+     * them here. */
+    if (steps > 0)
+    {
+        LOG((2, "Waiting for outstanding msgs"));
+        if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if (isend)
+            if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Provides the functionality of MPI_Gatherv with flow control
+ * options. This function is not currently used, but we hope it will
+ * be useful in future optimizations.
+ *
+ * @param sendbuf starting address of send buffer.
+ * @param sendcnt number of elements in send buffer.
+ * @param sendtype data type of send buffer elements.
+ * @param recvbuf address of receive buffer.
+ * @param recvcnts integer array (of length group size) containing the
+ * number of elements that are received from each process (significant
+ * only at root).
+ * @param displs integer array (of length group size). Entry i
+ * specifies the displacement relative to recvbuf at which to place
+ * the incoming data from process i (significant only at root).
+ * @param recvtype data type of recv buffer elements (significant only
+ * at root).
+ * @param root rank of receiving process.
+ * @param comm communicator.
+ * @param flow_cntl if non-zero, flow control will be used.
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_fc_gatherv(const void *sendbuf, int sendcnt, MPI_Datatype sendtype,
+                   void *recvbuf, const int *recvcnts, const int *displs,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm, int flow_cntl)
+{
+    bool fc_gather;
+    int gather_block_size;
+    int mytask, nprocs;
+    int mtag;
+    MPI_Status status;
+    int hs;
+    int dsize;
+    int mpierr;  /* Return code from MPI functions. */
+
+    if (flow_cntl > 0)
+    {
+        fc_gather = true;
+        gather_block_size = min(flow_cntl, MAX_GATHER_BLOCK_SIZE);
+    }
+    else
+    {
+        fc_gather = false;
+    }
+
+    if (fc_gather)
+    {
+        if ((mpierr = MPI_Comm_rank(comm, &mytask)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Comm_size(comm, &nprocs)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        mtag = 2 * nprocs;
+        hs = 1;
+
+        if (mytask == root)
+        {
+            int preposts = min(nprocs-1, gather_block_size);
+            int head = 0;
+            int count = 0;
+            int tail = 0;
+            MPI_Request rcvid[gather_block_size];
+
+            if ((mpierr = MPI_Type_size(recvtype, &dsize)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            for (int p = 0; p < nprocs; p++)
+            {
+                if (p != root)
+                {
+                    if (recvcnts[p] > 0)
+                    {
+                        count++;
+                        if (count > preposts)
+                        {
+                            if ((mpierr = MPI_Wait(rcvid + tail, &status)))
+                                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                            tail = (tail + 1) % preposts;
+                        }
+
+                        void *ptr = (void *)((char *)recvbuf + dsize * displs[p]);
+
+                        if ((mpierr = MPI_Irecv(ptr, recvcnts[p], recvtype, p, mtag, comm, rcvid + head)))
+                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                        head = (head + 1) % preposts;
+                        if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, mtag, comm)))
+                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    }
+                }
+            }
+
+            /* copy local data */
+            if ((mpierr = MPI_Type_size(sendtype, &dsize)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if ((mpierr = MPI_Sendrecv(sendbuf, sendcnt, sendtype, mytask, 102, recvbuf, recvcnts[mytask],
+                                       recvtype, mytask, 102, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            count = min(count, preposts);
+            if (count > 0)
+                if ((mpierr = MPI_Waitall(count, rcvid, MPI_STATUSES_IGNORE)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+        else
+        {
+            if (sendcnt > 0)
+            {
+                if ((mpierr = MPI_Recv(&hs, 1, MPI_INT, root, mtag, comm, &status)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(sendbuf, sendcnt, sendtype, root, mtag, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+        }
+    }
+    else
+    {
+        if ((mpierr = MPI_Gatherv(sendbuf, sendcnt, sendtype, recvbuf, recvcnts,
+                                  displs, recvtype, root, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_varm.c b/src/externals/pio2/src/clib/pio_varm.c
new file mode 100644
index 00000000000..18d55c5aea8
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_varm.c
@@ -0,0 +1,1811 @@
+#include 
+#include 
+#include 
+
+///
+/// PIO interface to nc_put_varm
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED_CHAR;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_CHAR;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_DOUBLE;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_CHAR;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_INT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibufcnt = bufcount;
+    ibuftype = buftype;
+    ierr = PIO_NOERR;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_FLOAT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_LONG;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED_SHORT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_LONG_LONG;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_SHORT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED_LONG_LONG;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync_interface)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async_interface || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
diff --git a/src/externals/pio2/src/clib/pioc.c b/src/externals/pio2/src/clib/pioc.c
new file mode 100644
index 00000000000..9612f65c8ee
--- /dev/null
+++ b/src/externals/pio2/src/clib/pioc.c
@@ -0,0 +1,1465 @@
+/**
+ * @file
+ * Some initialization and support functions.
+ * @author Jim Edwards
+ * @date  2014
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/** The default error handler used when iosystem cannot be located. */
+int default_error_handler = PIO_INTERNAL_ERROR;
+
+/** The target blocksize for each io task when the box rearranger is
+ * used (see pio_sc.c). */
+extern int blocksize;
+
+/**
+ * Check to see if PIO has been initialized.
+ *
+ * @param iosysid the IO system ID
+ * @param active pointer that gets true if IO system is active, false
+ * otherwise.
+ * @returns 0 on success, error code otherwise
+ */
+int PIOc_iosystem_is_active(int iosysid, bool *active)
+{
+    iosystem_desc_t *ios;
+
+    /* Get the ios if there is one. */
+    ios = pio_get_iosystem_from_id(iosysid);
+
+    if (active)
+    {
+        if (!ios || (ios->comp_comm == MPI_COMM_NULL && ios->io_comm == MPI_COMM_NULL))
+            *active = false;
+        else
+            *active = true;
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Check to see if PIO file is open.
+ *
+ * @param ncid the ncid of an open file
+ * @returns 1 if file is open, 0 otherwise.
+ */
+int PIOc_File_is_Open(int ncid)
+{
+    file_desc_t *file;
+
+    /* If get file returns non-zero, then this file is not open. */
+    if (pio_get_file(ncid, &file))
+        return 0;
+    else
+        return 1;
+}
+
+/**
+ * Set the error handling method to be used for subsequent pio library
+ * calls, returns the previous method setting. Note that this changes
+ * error handling for the IO system that was used when this file was
+ * opened. Other files opened with the same IO system will also he
+ * affected by this call. This function is supported but
+ * deprecated. New code should use PIOc_set_iosystem_error_handling().
+ * This method has no way to return an error, so any failure will
+ * result in MPI_Abort.
+ *
+ * @param ncid the ncid of an open file
+ * @param method the error handling method
+ * @returns old error handler
+ * @ingroup PIO_error_method
+ */
+int PIOc_Set_File_Error_Handling(int ncid, int method)
+{
+    file_desc_t *file;
+    int oldmethod;
+
+    /* Get the file info. */
+    if (pio_get_file(ncid, &file))
+        piodie("Could not find file", __FILE__, __LINE__);
+
+    /* Check that valid error handler was provided. */
+    if (method != PIO_INTERNAL_ERROR && method != PIO_BCAST_ERROR &&
+        method != PIO_RETURN_ERROR)
+        piodie("Invalid error hanlder method", __FILE__, __LINE__);
+
+    /* Get the old method. */
+    oldmethod = file->iosystem->error_handler;
+
+    /* Set the error hanlder. */
+    file->iosystem->error_handler = method;
+
+    return oldmethod;
+}
+
+/**
+ * Increment the unlimited dimension of the given variable.
+ *
+ * @param ncid the ncid of the open file
+ * @param varid the variable ID
+ * @returns 0 on success, error code otherwise
+ */
+int PIOc_advanceframe(int ncid, int varid)
+{
+    file_desc_t *file;
+    int ret;
+
+    /* Get the file info. */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (varid < 0 || varid >= PIO_MAX_VARS)
+        return pio_err(NULL, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    file->varlist[varid].record++;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set the unlimited dimension of the given variable
+ *
+ * @param ncid the ncid of the file.
+ * @param varid the varid of the variable
+ * @param frame the value of the unlimited dimension.  In c 0 for the
+ * first record, 1 for the second
+ * @return PIO_NOERR for no error, or error code.
+ * @ingroup PIO_setframe
+ */
+int PIOc_setframe(int ncid, int varid, int frame)
+{
+    file_desc_t *file;
+    int ret;
+
+    /* Get file info. */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (varid < 0 || varid >= PIO_MAX_VARS)
+        return pio_err(NULL, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    file->varlist[varid].record = frame;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the number of IO tasks set.
+ *
+ * @param iosysid the IO system ID
+ * @param numiotasks a pointer taht gets the number of IO
+ * tasks. Ignored if NULL.
+ * @returns 0 on success, error code otherwise
+ */
+int PIOc_get_numiotasks(int iosysid, int *numiotasks)
+{
+    iosystem_desc_t *ios;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (numiotasks)
+        *numiotasks = ios->num_iotasks;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the local size of the variable.
+ *
+ * @param ioid IO descrption ID.
+ * @returns the size of the array.
+ */
+int PIOc_get_local_array_size(int ioid)
+{
+    io_desc_t *iodesc;
+
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        piodie("Could not get iodesc", __FILE__, __LINE__);
+
+    return iodesc->ndof;
+}
+
+/**
+ * Set the error handling method used for subsequent calls. This
+ * function is deprecated. New code should use
+ * PIOc_set_iosystem_error_handling(). This method has no way to
+ * return an error, so any failure will result in MPI_Abort.
+ *
+ * @param iosysid the IO system ID
+ * @param method the error handling method
+ * @returns old error handler
+ * @ingroup PIO_error_method
+ */
+int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
+{
+    iosystem_desc_t *ios;
+    int oldmethod;
+
+    /* Get the iosystem info. */
+    if (iosysid != PIO_DEFAULT)
+        if (!(ios = pio_get_iosystem_from_id(iosysid)))
+            return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Set the error handler. */
+    if (PIOc_set_iosystem_error_handling(iosysid, method, &oldmethod))
+        piodie("Could not set the IOSystem error hanlder", __FILE__, __LINE__);
+
+    return oldmethod;
+}
+
+/**
+ * Set the error handling method used for subsequent calls for this IO
+ * system.
+ *
+ * @param iosysid the IO system ID. Passing PIO_DEFAULT instead
+ * changes the default error handling for the library.
+ * @param method the error handling method
+ * @param old_method pointer to int that will get old method. Ignored
+ * if NULL.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_error_method
+ */
+int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
+{
+    iosystem_desc_t *ios = NULL;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_set_iosystem_error_handling iosysid = %d method = %d", iosysid,
+         method));
+
+    /* Find info about this iosystem. */
+    if (iosysid != PIO_DEFAULT)
+        if (!(ios = pio_get_iosystem_from_id(iosysid)))
+            return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check that valid error handler was provided. */
+    if (method != PIO_INTERNAL_ERROR && method != PIO_BCAST_ERROR &&
+        method != PIO_RETURN_ERROR)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* If using async, and not an IO task, then send parameters. */
+    if (iosysid != PIO_DEFAULT)
+        if (ios->async_interface)
+        {
+            if (!ios->ioproc)
+            {
+                int msg = PIO_MSG_SETERRORHANDLING;
+                char old_method_present = old_method ? true : false;
+
+                if (ios->compmaster == MPI_ROOT)
+                    mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+                if (!mpierr)
+                    mpierr = MPI_Bcast(&method, 1, MPI_INT, ios->compmaster, ios->intercomm);
+                if (!mpierr)
+                    mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            }
+
+            /* Handle MPI errors. */
+            if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+                check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            if (mpierr)
+                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        }
+
+    /* Return the current handler. */
+    if (old_method)
+        *old_method = iosysid == PIO_DEFAULT ? default_error_handler : ios->error_handler;
+
+    /* Set new error handler. */
+    if (iosysid == PIO_DEFAULT)
+        default_error_handler = method;
+    else
+        ios->error_handler = method;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Initialize the decomposition used with distributed arrays. The
+ * decomposition describes how the data will be distributed between
+ * tasks.
+ *
+ * @param iosysid the IO system ID.
+ * @param basetype the basic PIO data type used.
+ * @param ndims the number of dimensions in the variable, not
+ * including the unlimited dimension.
+ * @param dims an array of global size of each dimension.
+ * @param maplen the local length of the compmap array.
+ * @param compmap a 1 based array of offsets into the array record on
+ * file. A 0 in this array indicates a value which should not be
+ * transfered.
+ * @param ioidp pointer that will get the io description ID.
+ * @param rearranger pointer to the rearranger to be used for this
+ * decomp or NULL to use the default.
+ * @param iostart An array of start values for block cyclic
+ * decompositions. If NULL ???
+ * @param iocount An array of count values for block cyclic
+ * decompositions. If NULL ???
+ * @returns 0 on success, error code otherwise
+ * @ingroup PIO_initdecomp
+ */
+int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int maplen,
+                    const PIO_Offset *compmap, int *ioidp, const int *rearranger,
+                    const PIO_Offset *iostart, const PIO_Offset *iocount)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    io_desc_t *iodesc;     /* The IO description. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+    int ierr;              /* Return code. */
+
+    LOG((1, "PIOc_InitDecomp iosysid = %d basetype = %d ndims = %d maplen = %d",
+         iosysid, basetype, ndims, maplen));
+
+    /* Get IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Caller must provide these. */
+    if (!dims || !compmap || !ioidp)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Check the dim lengths. */
+    for (int i = 0; i < ndims; i++)
+        if (dims[i] <= 0)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INITDECOMP_DOF; /* Message for async notification. */
+            char rearranger_present = rearranger ? true : false;
+            char iostart_present = iostart ? true : false;
+            char iocount_present = iocount ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&basetype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((int *)dims, ndims, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)compmap, maplen, MPI_OFFSET, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (rearranger_present && !mpierr)
+                mpierr = MPI_Bcast((int *)rearranger, 1, MPI_INT, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (iostart_present && !mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)iostart, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (iocount_present && !mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_InitDecomp iosysid = %d basetype = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d "
+                 "iocount_present = %d ", iosysid, basetype, ndims, maplen, rearranger_present, iostart_present, iocount_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Allocate space for the iodesc info. */
+    if ((ierr = malloc_iodesc(ios, basetype, ndims, &iodesc)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
+    /* Remember the maplen. */
+    iodesc->maplen = maplen;
+
+    /* Remember the map. */
+    if (!(iodesc->map = malloc(sizeof(PIO_Offset) * maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int m = 0; m < maplen; m++)
+        iodesc->map[m] = compmap[m];
+
+    /* Remember the dim sizes. */
+    if (!(iodesc->dimlen = malloc(sizeof(int) * ndims)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int d = 0; d < ndims; d++)
+        iodesc->dimlen[d] = dims[d];
+
+    /* Set the rearranger. */
+    if (!rearranger)
+        iodesc->rearranger = ios->default_rearranger;
+    else
+        iodesc->rearranger = *rearranger;
+    LOG((2, "iodesc->rearranger = %d", iodesc->rearranger));
+
+    /* Is this the subset rearranger? */
+    if (iodesc->rearranger == PIO_REARR_SUBSET)
+    {
+        LOG((2, "Handling subset rearranger."));
+        if (iostart && iocount)
+            fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp",
+                    "are incompatable with subset rearrange method and will be ignored");
+        iodesc->num_aiotasks = ios->num_iotasks;
+        if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, dims,
+                                            ndims, iodesc)))
+            return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    }
+    else
+    {
+        LOG((2, "Handling not the subset rearranger."));
+        if (ios->ioproc)
+        {
+            /*  Unless the user specifies the start and count for each
+             *  IO task compute it. */
+            if (iostart && iocount)
+            {
+                iodesc->maxiobuflen = 1;
+                for (int i = 0; i < ndims; i++)
+                {
+                    iodesc->firstregion->start[i] = iostart[i];
+                    iodesc->firstregion->count[i] = iocount[i];
+                    compute_maxIObuffersize(ios->io_comm, iodesc);
+
+                }
+                iodesc->num_aiotasks = ios->num_iotasks;
+            }
+            else
+            {
+                iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims,
+                                                         ios->num_iotasks, ios->io_rank,
+                                                         iodesc->firstregion->start, iodesc->firstregion->count);
+            }
+            compute_maxIObuffersize(ios->io_comm, iodesc);
+        }
+
+        /* Depending on array size and io-blocksize the actual number
+         * of io tasks used may vary. */
+        if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "iodesc->num_aiotasks = %d", iodesc->num_aiotasks));
+
+        /* Compute the communications pattern for this decomposition. */
+        if (iodesc->rearranger == PIO_REARR_BOX)
+            ierr = box_rearrange_create(ios, maplen, compmap, dims, ndims, iodesc);
+
+        /*
+          if (ios->ioproc){
+          io_region *ioregion = iodesc->firstregion;
+          while(ioregion != NULL){
+          for (int i=0;istart[i],ioregion->count[i]);
+          ioregion = ioregion->next;
+          }
+          }
+        */
+    }
+
+    /* Add this IO description to the list. */
+    *ioidp = pio_add_to_iodesc_list(iodesc);
+
+    LOG((3, "About to tune rearranger..."));
+    performance_tune_rearranger(ios, iodesc);
+    LOG((3, "Done with rearranger tune."));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Initialize the decomposition used with distributed arrays. The
+ * decomposition describes how the data will be distributed between
+ * tasks.
+ *
+ * @param iosysid the IO system ID.
+ * @param basetype the basic PIO data type used.
+ * @param ndims the number of dimensions in the variable, not
+ * including the unlimited dimension.
+ * @param dims an array of global size of each dimension.
+ * @param maplen the local length of the compmap array.
+ * @param compmap a 0 based array of offsets into the array record on
+ * file. A -1 in this array indicates a value which should not be
+ * transfered.
+ * @param ioidp pointer that will get the io description ID.
+ * @param rearranger pointer to the rearranger to be used for this
+ * decomp or NULL to use the default.
+ * @param iostart An array of start values for block cyclic
+ * decompositions. If NULL ???
+ * @param iocount An array of count values for block cyclic
+ * decompositions. If NULL ???
+ * @returns 0 on success, error code otherwise
+ * @ingroup PIO_initdecomp
+ */
+int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int maplen,
+                     const PIO_Offset *compmap, int *ioidp, const int *rearranger,
+                     const PIO_Offset *iostart, const PIO_Offset *iocount)
+{
+    PIO_Offset compmap_1_based[maplen];
+
+    LOG((1, "PIOc_init_decomp iosysid = %d basetype = %d ndims = %d maplen = %d",
+         iosysid, basetype, ndims, maplen));
+
+    /* Add 1 to all elements in compmap. */
+    for (int e = 0; e < maplen; e++)
+        compmap_1_based[e] = compmap[e] + 1;
+
+    return PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap_1_based,
+                           ioidp, rearranger, iostart, iocount);
+}
+
+/**
+ * This is a simplified initdecomp which can be used if the memory
+ * order of the data can be expressed in terms of start and count on
+ * the file. In this case we compute the compdof.
+ *
+ * @param iosysid the IO system ID
+ * @param basetype
+ * @param ndims the number of dimensions
+ * @param dims array of dimensions
+ * @param start start array
+ * @param count count array
+ * @param pointer that gets the IO ID.
+ * @returns 0 for success, error code otherwise
+ * @ingroup PIO_initdecomp
+ */
+int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims,
+                       const long int *start, const long int *count, int *ioidp)
+
+{
+    iosystem_desc_t *ios;
+    int n, i, maplen = 1;
+    PIO_Offset prod[ndims], loc[ndims];
+    int rearr = PIO_REARR_SUBSET;
+
+    LOG((1, "PIOc_InitDecomp_bc iosysid = %d basetype = %d ndims = %d"));
+
+    /* Get the info about the io system. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check for required inputs. */
+    if (!dims || !start || !count || !ioidp)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Check that dim, start, and count values are not obviously
+     * incorrect. */
+    for (int i = 0; i < ndims; i++)
+        if (dims[i] <= 0 || start[i] < 0 || count[i] < 0 || (start[i] + count[i]) > dims[i])
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Find the maplen. */
+    for (i = 0; i < ndims; i++)
+        maplen *= count[i];
+
+    /* Get storage for the compmap. */
+    PIO_Offset compmap[maplen];
+
+    /* Find the compmap. */
+    prod[ndims - 1] = 1;
+    loc[ndims - 1] = 0;
+    for (n = ndims - 2; n >= 0; n--)
+    {
+        prod[n] = prod[n + 1] * dims[n + 1];
+        loc[n] = 0;
+    }
+    for (i = 0; i < maplen; i++)
+    {
+        compmap[i] = 0;
+        for (n = ndims - 1; n >= 0; n--)
+            compmap[i] += (start[n] + loc[n]) * prod[n];
+
+        n = ndims - 1;
+        loc[n] = (loc[n] + 1) % count[n];
+        while (loc[n] == 0 && n > 0)
+        {
+            n--;
+            loc[n] = (loc[n] + 1) % count[n];
+        }
+    }
+
+    return PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap, ioidp,
+                           &rearr, NULL, NULL);
+}
+
+/**
+ * Library initialization used when IO tasks are a subset of compute
+ * tasks.
+ *
+ * This function creates an MPI intracommunicator between a set of IO
+ * tasks and one or more sets of computational tasks.
+ *
+ * The caller must create all comp_comm and the io_comm MPI
+ * communicators before calling this function.
+ *
+ * @param comp_comm the MPI_Comm of the compute tasks
+ * @param num_iotasks the number of io tasks to use
+ * @param stride the offset between io tasks in the comp_comm
+ * @param base the comp_comm index of the first io task
+ * @param rearr the rearranger to use by default, this may be
+ * overriden in the @ref PIO_initdecomp
+ * @param iosysidp index of the defined system descriptor
+ * @return 0 on success, otherwise a PIO error code.
+ * @ingroup PIO_init
+ */
+int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base,
+                        int rearr, int *iosysidp)
+{
+    iosystem_desc_t *ios;
+    int ustride;
+    int num_comptasks; /* The size of the comp_comm. */
+    int mpierr;        /* Return value for MPI calls. */
+    int ret;           /* Return code for function calls. */
+
+    /* Turn on the logging system. */
+    pio_init_logging();
+
+    /* Find the number of computation tasks. */
+    if ((mpierr = MPI_Comm_size(comp_comm, &num_comptasks)))
+        return check_mpi2(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Check the inputs. */
+    if (!iosysidp || num_iotasks < 1 || num_iotasks * stride > num_comptasks)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_Init_Intracomm comp_comm = %d num_iotasks = %d stride = %d base = %d "
+         "rearr = %d", comp_comm, num_iotasks, stride, base, rearr));
+
+    /* Allocate memory for the iosystem info. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    ios->io_comm = MPI_COMM_NULL;
+    ios->intercomm = MPI_COMM_NULL;
+    ios->error_handler = default_error_handler;
+    ios->default_rearranger = rearr;
+    ios->num_iotasks = num_iotasks;
+    ios->num_comptasks = num_comptasks;
+
+    /* Initialize the rearranger options. */
+    init_rearr_opts(ios);
+
+    /* Copy the computation communicator into union_comm. */
+    if ((mpierr = MPI_Comm_dup(comp_comm, &ios->union_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Copy the computation communicator into comp_comm. */
+    if ((mpierr = MPI_Comm_dup(comp_comm, &ios->comp_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "union_comm = %d comp_comm = %d", ios->union_comm, ios->comp_comm));
+
+    ios->my_comm = ios->comp_comm;
+    ustride = stride;
+
+    /* Find MPI rank comp_comm communicator. */
+    if ((mpierr = MPI_Comm_rank(ios->comp_comm, &ios->comp_rank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (ios->comp_rank == 0)
+        ios->compmaster = MPI_ROOT;
+    LOG((2, "comp_rank = %d num_comptasks = %d", ios->comp_rank, ios->num_comptasks));
+
+    /* Create an array that holds the ranks of the tasks to be used
+     * for IO. NOTE that sizeof(int) should probably be 1, not
+     * sizeof(int) ???*/
+    if (!(ios->ioranks = calloc(sizeof(int), ios->num_iotasks)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        ios->ioranks[i] = (base + i * ustride) % ios->num_comptasks;
+        if (ios->ioranks[i] == ios->comp_rank)
+            ios->ioproc = true;
+    }
+    ios->ioroot = ios->ioranks[0];
+
+    for (int i = 0; i < ios->num_iotasks; i++)
+        LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i]));
+
+    /* We are not providing an info object. */
+    ios->info = MPI_INFO_NULL;
+
+    /* The task that has an iomaster value of MPI_ROOT will be the
+     * root of the IO communicator. */
+    if (ios->comp_rank == ios->ioranks[0])
+        ios->iomaster = MPI_ROOT;
+
+    /* Create a group for the computation tasks. */
+    if ((mpierr = MPI_Comm_group(ios->comp_comm, &ios->compgroup)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Create a group for the IO tasks. */
+    if ((mpierr = MPI_Group_incl(ios->compgroup, ios->num_iotasks, ios->ioranks,
+                                 &ios->iogroup)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Create an MPI communicator for the IO tasks. */
+    if ((mpierr = MPI_Comm_create(ios->comp_comm, ios->iogroup, &ios->io_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* For the tasks that are doing IO, get their rank within the IO
+     * communicator. For some reason when I check the return value of
+     * this MPI call, all tests start to fail! */
+    if (ios->ioproc)
+    {
+        if ((mpierr = MPI_Comm_rank(ios->io_comm, &ios->io_rank)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    else
+        ios->io_rank = -1;
+    LOG((3, "ios->io_comm = %d ios->io_rank = %d", ios->io_comm, ios->io_rank));
+
+    ios->union_rank = ios->comp_rank;
+
+    /* Add this ios struct to the list in the PIO library. */
+    *iosysidp = pio_add_to_iosystem_list(ios);
+
+    /* Allocate buffer space for compute nodes. */
+    if ((ret = compute_buffer_init(ios)))
+        return ret;
+
+    LOG((2, "Init_Intracomm complete iosysid = %d", *iosysidp));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Interface to call from pio_init from fortran.
+ *
+ * @param f90_comp_comm
+ * @param num_iotasks the number of IO tasks
+ * @param stride the stride to use assigning tasks
+ * @param base the starting point when assigning tasks
+ * @param rearr the rearranger
+ * @param rearr_opts the rearranger options
+ * @param iosysidp a pointer that gets the IO system ID
+ * @returns 0 for success, error code otherwise
+ */
+int PIOc_Init_Intracomm_from_F90(int f90_comp_comm,
+                                 const int num_iotasks, const int stride,
+                                 const int base, const int rearr,
+                                 rearr_opt_t *rearr_opts, int *iosysidp)
+{
+    int ret = PIO_NOERR;
+    ret = PIOc_Init_Intracomm(MPI_Comm_f2c(f90_comp_comm), num_iotasks,
+                              stride, base, rearr,
+                              iosysidp);
+    if (ret != PIO_NOERR)
+    {
+        LOG((1, "PIOc_Init_Intracomm failed"));
+        return ret;
+    }
+
+    if (rearr_opts)
+    {
+        LOG((1, "Setting rearranger options, iosys=%d", *iosysidp));
+        return PIOc_set_rearr_opts(*iosysidp, rearr_opts->comm_type,
+                                   rearr_opts->fcd,
+                                   rearr_opts->comm_fc_opts_comp2io.enable_hs,
+                                   rearr_opts->comm_fc_opts_comp2io.enable_isend,
+                                   rearr_opts->comm_fc_opts_comp2io.max_pend_req,
+                                   rearr_opts->comm_fc_opts_io2comp.enable_hs,
+                                   rearr_opts->comm_fc_opts_io2comp.enable_isend,
+                                   rearr_opts->comm_fc_opts_io2comp.max_pend_req);
+    }
+    return ret;
+}
+
+/**
+ * Send a hint to the MPI-IO library.
+ *
+ * @param iosysid the IO system ID
+ * @param hint the hint for MPI
+ * @param hintval the value of the hint
+ * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ */
+int PIOc_set_hint(int iosysid, const char *hint, const char *hintval)
+{
+    iosystem_desc_t *ios;
+    int mpierr; /* Return value for MPI calls. */
+
+    /* Get the iosysid. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* User must provide these. */
+    if (!hint || !hintval)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_set_hint hint = %s hintval = %s", hint, hintval));
+
+    /* Make sure we have an info object. */
+    if (ios->info == MPI_INFO_NULL)
+        if ((mpierr = MPI_Info_create(&ios->info)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Set the MPI hint. */
+    if (ios->ioproc)
+        if ((mpierr = MPI_Info_set(ios->info, hint, hintval)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Clean up internal data structures, free MPI resources, and exit the
+ * pio library.
+ *
+ * @param iosysid: the io system ID provided by PIOc_Init_Intracomm().
+ * @returns 0 for success or non-zero for error.
+ * @ingroup PIO_finalize
+ */
+int PIOc_finalize(int iosysid)
+{
+    iosystem_desc_t *ios;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;
+
+    LOG((1, "PIOc_finalize iosysid = %d MPI_COMM_NULL = %d", iosysid,
+         MPI_COMM_NULL));
+
+    /* Find the IO system information. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If asynch IO is in use, send the PIO_MSG_EXIT message from the
+     * comp master to the IO processes. This may be called by
+     * componets for other components iosysid. So don't send unless
+     * there is a valid union_comm. */
+    if (ios->async_interface && ios->union_comm != MPI_COMM_NULL)
+    {
+        int msg = PIO_MSG_EXIT;
+
+        LOG((3, "found iosystem info comproot = %d union_comm = %d comp_idx = %d",
+             ios->comproot, ios->union_comm, ios->comp_idx));
+        if (!ios->ioproc)
+        {
+            LOG((2, "sending msg = %d ioroot = %d union_comm = %d", msg,
+                 ios->ioroot, ios->union_comm));
+
+            /* Send the message to the message handler. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the parameters of the function call. */
+            if (!mpierr)
+                mpierr = MPI_Bcast((int *)&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        LOG((3, "handling async errors mpierr = %d my_comm = %d", mpierr, ios->my_comm));
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "async errors bcast"));
+    }
+
+    /* Free this memory that was allocated in init_intracomm. */
+    if (ios->ioranks)
+        free(ios->ioranks);
+    LOG((3, "Freed ioranks."));
+
+    /* Free the buffer pool. */
+    int niosysid;
+    if ((ierr = pio_num_iosystem(&niosysid)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    LOG((2, "%d iosystems are still open.", niosysid));
+
+    /* Only free the buffer pool if this is the last open iosysid. */
+    if (niosysid == 1)
+    {
+        free_cn_buffer_pool(ios);
+        LOG((2, "Freed buffer pool."));
+    }
+
+    /* Free the MPI groups. */
+    if (ios->compgroup != MPI_GROUP_NULL)
+        MPI_Group_free(&ios->compgroup);
+
+    if (ios->iogroup != MPI_GROUP_NULL)
+        MPI_Group_free(&(ios->iogroup));
+
+    /* Free the MPI communicators. my_comm is just a copy (but not an
+     * MPI copy), so does not have to have an MPI_Comm_free()
+     * call. comp_comm and io_comm are MPI duplicates of the comms
+     * handed into init_intercomm. So they need to be freed by MPI. */
+    if (ios->intercomm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->intercomm);
+    if (ios->union_comm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->union_comm);
+    if (ios->io_comm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->io_comm);
+    if (ios->comp_comm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->comp_comm);
+    if (ios->my_comm != MPI_COMM_NULL)
+        ios->my_comm = MPI_COMM_NULL;
+
+    /* Free the MPI Info object. */
+    if (ios->info != MPI_INFO_NULL)
+        MPI_Info_free(&ios->info);
+
+    /* Delete the iosystem_desc_t data associated with this id. */
+    LOG((2, "About to delete iosysid %d.", iosysid));
+    if ((ierr = pio_delete_iosystem_from_list(iosysid)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+
+    pio_finalize_logging();
+
+    LOG((2, "PIOc_finalize completed successfully"));
+    return PIO_NOERR;
+}
+
+/**
+ * Return a logical indicating whether this task is an IO task.
+ *
+ * @param iosysid the io system ID
+ * @param ioproc a pointer that gets 1 if task is an IO task, 0
+ * otherwise. Ignored if NULL.
+ * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ */
+int PIOc_iam_iotask(int iosysid, bool *ioproc)
+{
+    iosystem_desc_t *ios;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (ioproc)
+        *ioproc = ios->ioproc;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Return the rank of this task in the IO communicator or -1 if this
+ * task is not in the communicator.
+ *
+ * @param iosysid the io system ID
+ * @param iorank a pointer that gets the io rank, or -1 if task is not
+ * in the IO communicator. Ignored if NULL.
+ * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ */
+int PIOc_iotask_rank(int iosysid, int *iorank)
+{
+    iosystem_desc_t *ios;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (iorank)
+        *iorank = ios->io_rank;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Return true if this iotype is supported in the build, 0 otherwise.
+ *
+ * @param iotype the io type to check
+ * @returns 1 if iotype is in build, 0 if not.
+ */
+int PIOc_iotype_available(int iotype)
+{
+    switch(iotype)
+    {
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+    case PIO_IOTYPE_NETCDF4C:
+        return 1;
+#endif
+    case PIO_IOTYPE_NETCDF:
+        return 1;
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+        return 1;
+        break;
+#endif
+    default:
+        return 0;
+    }
+}
+
+/**
+ * Library initialization used when IO tasks are distinct from compute
+ * tasks.
+ *
+ * This is a collective call.  Input parameters are read on
+ * comp_rank=0 values on other tasks are ignored.  This variation of
+ * PIO_init sets up a distinct set of tasks to handle IO, these tasks
+ * do not return from this call.  Instead they go to an internal loop
+ * and wait to receive further instructions from the computational
+ * tasks.
+ *
+ * Sequence of Events to do Asynch I/O
+ * -----------------------------------
+ *
+ * Here is the sequence of events that needs to occur when an IO
+ * operation is called from the collection of compute tasks.  I'm
+ * going to use pio_put_var because write_darray has some special
+ * characteristics that make it a bit more complicated...
+ *
+ * Compute tasks call pio_put_var with an integer argument
+ *
+ * The MPI_Send sends a message from comp_rank=0 to io_rank=0 on
+ * union_comm (a comm defined as the union of io and compute tasks)
+ * msg is an integer which indicates the function being called, in
+ * this case the msg is PIO_MSG_PUT_VAR_INT
+ *
+ * The iotasks now know what additional arguments they should expect
+ * to receive from the compute tasks, in this case a file handle, a
+ * variable id, the length of the array and the array itself.
+ *
+ * The iotasks now have the information they need to complete the
+ * operation and they call the pio_put_var routine.  (In pio1 this bit
+ * of code is in pio_get_put_callbacks.F90.in)
+ *
+ * After the netcdf operation is completed (in the case of an inq or
+ * get operation) the result is communicated back to the compute
+ * tasks.
+ *
+ * @param world the communicator containing all the available tasks.
+ *
+ * @param num_io_procs the number of processes for the IO component.
+ *
+ * @param io_proc_list an array of lenth num_io_procs with the
+ * processor number for each IO processor. If NULL then the IO
+ * processes are assigned starting at processes 0.
+ *
+ * @param component_count number of computational components
+ *
+ * @param num_procs_per_comp an array of int, of length
+ * component_count, with the number of processors in each computation
+ * component.
+ *
+ * @param proc_list an array of arrays containing the processor
+ * numbers for each computation component. If NULL then the
+ * computation components are assigned processors sequentially
+ * starting with processor num_io_procs.
+ *
+ * @param user_io_comm pointer to an MPI_Comm. If not NULL, it will
+ * get an MPI duplicate of the IO communicator. (It is a full
+ * duplicate and later must be freed with MPI_Free() by the caller.)
+ *
+ * @param user_comp_comm pointer to an array of pointers to MPI_Comm;
+ * the array is of length component_count. If not NULL, it will get an
+ * MPI duplicate of each computation communicator. (These are full
+ * duplicates and each must later be freed with MPI_Free() by the
+ * caller.)
+ *
+ * @param iosysidp pointer to array of length component_count that
+ * gets the iosysid for each component.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ * @ingroup PIO_init
+ */
+int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list,
+                    int component_count, int *num_procs_per_comp, int **proc_list,
+                    MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int *iosysidp)
+{
+    int my_rank;
+    int **my_proc_list;
+    int *my_io_proc_list;
+    int mpierr;
+    int ret;
+
+    /* Check input parameters. */
+    if (num_io_procs < 1 || component_count < 1 || !num_procs_per_comp || !iosysidp)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Temporarily limit to one computational component. */
+    if (component_count > 1)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Turn on the logging system for PIO. */
+    pio_init_logging();
+    LOG((1, "PIOc_Init_Async component_count = %d", component_count));
+
+    /* If the user did not supply a list of process numbers to use for
+     * IO, create it. */
+    if (!io_proc_list)
+    {
+        if (!(my_io_proc_list = malloc(num_io_procs * sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int p = 0; p < num_io_procs; p++)
+            my_io_proc_list[p] = p;
+    }
+    else
+        my_io_proc_list = io_proc_list;
+
+    /* If the user did not provide a list of processes for each
+     * component, create one. */
+    if (!proc_list)
+    {
+        int last_proc = 0;
+
+        /* Allocate space for array of arrays. */
+        if (!(my_proc_list = malloc((component_count + 1) * sizeof(int *))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Fill the array of arrays. */
+        for (int cmp = 0; cmp < component_count + 1; cmp++)
+        {
+            LOG((3, "calculating processors for component %d", cmp));
+
+            /* Allocate space for each array. */
+            if (!(my_proc_list[cmp] = malloc(num_procs_per_comp[cmp] * sizeof(int))))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+            int proc;
+            for (proc = last_proc; proc < num_procs_per_comp[cmp] + last_proc; proc++)
+            {
+                my_proc_list[cmp][proc - last_proc] = proc;
+                LOG((3, "my_proc_list[%d][%d] = %d", cmp, proc - last_proc, proc));
+            }
+            last_proc = proc;
+        }
+    }
+    else
+        my_proc_list = proc_list;
+
+    /* Get rank of this task. */
+    if ((ret = MPI_Comm_rank(world, &my_rank)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    /* Is this process in the IO component? */
+    int pidx;
+    for (pidx = 0; pidx < num_procs_per_comp[0]; pidx++)
+        if (my_rank == my_proc_list[0][pidx])
+            break;
+    int in_io = (pidx == num_procs_per_comp[0]) ? 0 : 1;
+    LOG((3, "in_io = %d", in_io));
+
+    /* Allocate struct to hold io system info for each computation component. */
+    /* Allocate struct to hold io system info for each component. */
+    iosystem_desc_t *iosys[component_count], *my_iosys;
+    for (int cmp1 = 0; cmp1 < component_count; cmp1++)
+        if (!(iosys[cmp1] = (iosystem_desc_t *)calloc(1, sizeof(iosystem_desc_t))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Create group for world. */
+    MPI_Group world_group;
+    if ((ret = MPI_Comm_group(world, &world_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+    LOG((3, "world group created\n"));
+
+    /* We will create a group for the IO component. */
+    MPI_Group io_group;
+
+    /* The shared IO communicator. */
+    MPI_Comm io_comm;
+
+    /* Rank of current process in IO communicator. */
+    int io_rank = -1;
+
+    /* Set to MPI_ROOT on master process, MPI_PROC_NULL on other
+     * processes. */
+    int iomaster;
+
+    /* Create a group for the IO component. */
+    if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+    LOG((3, "created IO group - io_group = %d group empty is %d", io_group, MPI_GROUP_EMPTY));
+    for (int p = 0; p < num_io_procs; p++)
+        LOG((3, "my_io_proc_list[%d] = %d", p, my_io_proc_list[p]));
+
+    /* There is one shared IO comm. Create it. */
+    if ((ret = MPI_Comm_create(world, io_group, &io_comm)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+    LOG((3, "created io comm io_comm = %d", io_comm));
+
+    /* Does the user want a copy of the IO communicator? */
+    if (user_io_comm)
+    {
+        *user_io_comm = MPI_COMM_NULL;
+        if (in_io)
+            if ((mpierr = MPI_Comm_dup(io_comm, user_io_comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* For processes in the IO component, get their rank within the IO
+     * communicator. */
+    if (in_io)
+    {
+        LOG((3, "about to get io rank"));
+        if ((ret = MPI_Comm_rank(io_comm, &io_rank)))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL;
+        LOG((3, "intracomm created for io_comm = %d io_rank = %d IO %s",
+             io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT"));
+    }
+
+    /* We will create a group for each component. */
+    MPI_Group group[component_count + 1];
+
+    /* We will also create a group for each component and the IO
+     * component processes (i.e. a union of computation and IO
+     * processes. */
+    MPI_Group union_group[component_count];
+
+    /* For each component, starting with the IO component. */
+    for (int cmp = 0; cmp < component_count + 1; cmp++)
+    {
+        LOG((3, "processing component %d", cmp));
+
+        /* Don't start initing iosys until after IO component. */
+        if (cmp)
+        {
+            /* Get pointer to current iosys. */
+            my_iosys = iosys[cmp - 1];
+
+            /* Initialize some values. */
+            my_iosys->io_comm = MPI_COMM_NULL;
+            my_iosys->comp_comm = MPI_COMM_NULL;
+            my_iosys->union_comm = MPI_COMM_NULL;
+            my_iosys->intercomm = MPI_COMM_NULL;
+            my_iosys->my_comm = MPI_COMM_NULL;
+            my_iosys->async_interface = 1;
+            my_iosys->error_handler = default_error_handler;
+            my_iosys->num_comptasks = num_procs_per_comp[cmp];
+            my_iosys->num_iotasks = num_procs_per_comp[0];
+            my_iosys->compgroup = MPI_GROUP_NULL;
+            my_iosys->iogroup = MPI_GROUP_NULL;
+
+            /* The rank of the computation leader in the union comm. */
+            my_iosys->comproot = num_procs_per_comp[0];
+            LOG((3, "my_iosys->comproot = %d", my_iosys->comproot));
+
+            /* We are not providing an info object. */
+            my_iosys->info = MPI_INFO_NULL;
+        }
+
+        /* Create a group for this component. */
+        if ((ret = MPI_Group_incl(world_group, num_procs_per_comp[cmp], my_proc_list[cmp],
+                                  &group[cmp])))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        LOG((3, "created component MPI group - group[%d] = %d", cmp, group[cmp]));
+
+        /* For all the computation components (i.e. cmp != 0), create
+         * a union group with their processors and the processors of
+         * the (shared) IO component. */
+        if (cmp)
+        {
+            /* How many processors in the union comm? */
+            int nprocs_union = num_procs_per_comp[0] + num_procs_per_comp[cmp];
+
+            /* This will hold proc numbers from both computation and IO
+             * components. */
+            int proc_list_union[nprocs_union];
+
+            /* Add proc numbers from IO. */
+            for (int p = 0; p < num_procs_per_comp[0]; p++)
+                proc_list_union[p] = my_proc_list[0][p];
+
+            /* Add proc numbers from computation component. */
+            for (int p = 0; p < num_procs_per_comp[cmp]; p++)
+                proc_list_union[p + num_procs_per_comp[0]] = my_proc_list[cmp][p];
+
+            /* Create the union group. */
+            if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union,
+                                      &union_group[cmp - 1])))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+            LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp, union_group[cmp-1], nprocs_union));
+        }
+
+        /* Remember whether this process is in the IO component. */
+        if (cmp)
+            my_iosys->ioproc = in_io;
+
+        /* Is this process in this computation component (which is the
+         * IO component if cmp == 0)? */
+        int in_cmp = 0;
+        for (pidx = 0; pidx < num_procs_per_comp[cmp]; pidx++)
+            if (my_rank == my_proc_list[cmp][pidx])
+                break;
+        in_cmp = (pidx == num_procs_per_comp[cmp]) ? 0 : 1;
+        LOG((3, "pidx = %d num_procs_per_comp[%d] = %d in_cmp = %d",
+             pidx, cmp, num_procs_per_comp[cmp], in_cmp));
+
+        /* Create an intracomm for this component. Only processes in
+         * the component need to participate in the intracomm create
+         * call. */
+        /* Create the intracomm from the group. */
+        LOG((3, "creating intracomm cmp = %d from group[%d] = %d", cmp, cmp, group[cmp]));
+
+        /* We handle the IO comm differently (cmp == 0). */
+        if (!cmp)
+        {
+            /* LOG((3, "about to create io comm")); */
+            /* if ((ret = MPI_Comm_create_group(world, group[cmp], cmp, &io_comm))) */
+            /*     return check_mpi(NULL, ret, __FILE__, __LINE__); */
+            /* LOG((3, "about to get io rank"));                 */
+            /* if ((ret = MPI_Comm_rank(io_comm, &io_rank))) */
+            /*     return check_mpi(NULL, ret, __FILE__, __LINE__); */
+            /* iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL; */
+            /* LOG((3, "intracomm created for cmp = %d io_comm = %d io_rank = %d IO %s", */
+            /*      cmp, io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT")); */
+        }
+        else
+        {
+            if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm)))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+            if (in_cmp)
+            {
+                /* Does the user want a copy? */
+                if (user_comp_comm)
+                    if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp - 1])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                /* Get the rank in this comp comm. */
+                if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank)))
+                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+                /* Set comp_rank 0 to be the compmaster. It will have
+                 * a setting of MPI_ROOT, all other tasks will have a
+                 * setting of MPI_PROC_NULL. */
+                my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT;
+
+                LOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s",
+                     cmp, my_iosys->comp_comm, my_iosys->comp_rank,
+                     my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT"));
+            }
+        }
+
+
+        /* If this is the IO component, make a copy of the IO comm for
+         * each computational component. */
+        if (in_io)
+            if (cmp)
+            {
+                LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank));
+                if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm)))
+                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+                LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank));
+                my_iosys->iomaster = iomaster;
+                my_iosys->io_rank = io_rank;
+                my_iosys->ioroot = 0;
+                my_iosys->comp_idx = cmp - 1;
+            }
+
+        /* All the processes in this component, and the IO component,
+         * are part of the union_comm. */
+        if (cmp)
+        {
+            if (in_io || in_cmp)
+            {
+                LOG((3, "my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp-1]));
+                /* Create a group for the union of the IO component
+                 * and one of the computation components. */
+                if ((ret = MPI_Comm_create(world, union_group[cmp - 1],
+                                           &my_iosys->union_comm)))
+                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+                if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank)))
+                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+                /* Set my_comm to union_comm for async. */
+                my_iosys->my_comm = my_iosys->union_comm;
+                LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d",
+                     cmp, my_iosys->union_rank, my_iosys->union_comm));
+
+                if (in_io)
+                {
+                    LOG((3, "my_iosys->io_comm = %d", my_iosys->io_comm));
+                    /* Create the intercomm from IO to computation component. */
+                    LOG((3, "about to create intercomm for IO component to cmp = %d "
+                         "my_iosys->io_comm = %d", cmp, my_iosys->io_comm));
+                    if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm,
+                                                    my_proc_list[cmp][0], 0, &my_iosys->intercomm)))
+                        return check_mpi(NULL, ret, __FILE__, __LINE__);
+                }
+                else
+                {
+                    /* Create the intercomm from computation component to IO component. */
+                    LOG((3, "about to create intercomm for cmp = %d my_iosys->comp_comm = %d", cmp,
+                         my_iosys->comp_comm));
+                    if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm,
+                                                    my_proc_list[0][0], 0, &my_iosys->intercomm)))
+                        return check_mpi(NULL, ret, __FILE__, __LINE__);
+                }
+                LOG((3, "intercomm created for cmp = %d", cmp));
+            }
+
+            /* Add this id to the list of PIO iosystem ids. */
+            iosysidp[cmp - 1] = pio_add_to_iosystem_list(my_iosys);
+            LOG((2, "new iosys ID added to iosystem_list iosysid = %d\n", iosysidp[cmp - 1]));
+        }
+    }
+
+    /* Now call the function from which the IO tasks will not return
+     * until the PIO_MSG_EXIT message is sent. This will handle all
+     * components. */
+    if (in_io)
+    {
+        LOG((2, "Starting message handler io_rank = %d component_count = %d",
+             io_rank, component_count));
+        if ((ret = pio_msg_handler2(io_rank, component_count, iosys, io_comm)))
+            return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+        LOG((2, "Returned from pio_msg_handler2() ret = %d", ret));
+    }
+
+    /* Free resources if needed. */
+    LOG((2, "PIOc_Init_Async starting to free resources"));
+    if (!io_proc_list)
+        free(my_io_proc_list);
+
+    if (in_io)
+        if ((mpierr = MPI_Comm_free(&io_comm)))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    if (!proc_list)
+    {
+        for (int cmp = 0; cmp < component_count + 1; cmp++)
+            free(my_proc_list[cmp]);
+        free(my_proc_list);
+    }
+
+    /* Free MPI groups. */
+    if ((ret = MPI_Group_free(&io_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    for (int cmp = 0; cmp < component_count + 1; cmp++)
+    {
+        if ((ret = MPI_Group_free(&group[cmp])))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        if (cmp)
+            if ((ret = MPI_Group_free(&union_group[cmp - 1])))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+    }
+
+    if ((ret = MPI_Group_free(&world_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    LOG((2, "successfully done with PIO_Init_Async"));
+    return PIO_NOERR;
+}
+
+/**
+ * Set the target blocksize for the box rearranger.
+ *
+ * @param newblocksize the new blocksize.
+ * @returns 0 for success.
+ * @ingroup PIO_set_blocksize
+ */
+int PIOc_set_blocksize(int newblocksize)
+{
+    if (newblocksize > 0)
+        blocksize = newblocksize;
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pioc_sc.c b/src/externals/pio2/src/clib/pioc_sc.c
new file mode 100644
index 00000000000..6b991b373db
--- /dev/null
+++ b/src/externals/pio2/src/clib/pioc_sc.c
@@ -0,0 +1,381 @@
+/**
+ * @file
+ * Compute start and count arrays for the box rearranger
+ *
+ * @author Jim Edwards
+ * @date  2014
+ */
+#include 
+#include 
+#include 
+
+/** The default target blocksize for each io task when the box
+ * rearranger is used. */
+#define DEFAULT_BLOCKSIZE 1024
+
+/** The target blocksize for each io task when the box rearranger is
+ * used. */
+int blocksize = DEFAULT_BLOCKSIZE;
+
+/**
+ * Recursive Standard C Function: Greatest Common Divisor.
+ *
+ * @param a
+ * @param b
+ * @returns greates common divisor.
+ */
+int gcd(int a, int b )
+{
+    if (a == 0)
+        return b;
+    return gcd(b % a, a);
+}
+
+/**
+ * Recursive Standard C Function: Greatest Common Divisor for 64 bit
+ * ints.
+ *
+ * @param a
+ * @param b
+ * @returns greates common divisor.
+ */
+long long lgcd(long long a, long long b)
+{
+    if (a == 0)
+        return b;
+    return lgcd(b % a, a);
+}
+
+/**
+ * Return the gcd of elements in an int array.
+ *
+ * @param nain length of the array
+ * @param ain an array of length nain
+ * @returns greatest common divisor.
+ */
+int gcd_array(int nain, int *ain)
+{
+    int i;
+    int bsize = 1;
+
+    for (i = 0; i < nain; i++)
+        if (ain[i] <= 1)
+            return bsize;
+
+    bsize = ain[0];
+    i = 1;
+    while (i < nain && bsize > 1)
+    {
+        bsize = gcd(bsize, ain[i]);
+        i++;
+    }
+
+    return bsize;
+}
+
+/**
+ * Return the gcd of nain and any value in ain as int_64.
+ *
+ * @param main
+ * @param ain
+ * @returns
+ */
+long long lgcd_array(int nain, long long *ain)
+{
+    int i;
+    long long bsize = 1;
+
+    for (i = 0; i < nain; i++)
+        if(ain[i] <= 1)
+            return bsize;
+
+    bsize = ain[0];
+    i = 1;
+    while (i < nain && bsize > 1)
+    {
+        bsize = gcd(bsize, ain[i]);
+        i++;
+    }
+
+    return bsize;
+}
+
+/**
+ * Compute start and count arrays.
+ *
+ * @param gdim
+ * @param ioprocs
+ * @param rank
+ * @param start
+ * @param kount
+ */
+void computestartandcount(int gdim, int ioprocs, int rank, PIO_Offset *start,
+                          PIO_Offset *kount)
+{
+    int irank;
+    int remainder;
+    int adds;
+    PIO_Offset lstart, lkount;
+
+    irank = rank % ioprocs;
+    lkount = (long int)(gdim / ioprocs);
+    lstart = (long int)(lkount * irank);
+    remainder = gdim - lkount * ioprocs;
+
+    if (remainder >= ioprocs-irank)
+    {
+        lkount++;
+        if ((adds = irank + remainder - ioprocs) > 0)
+            lstart += adds;
+    }
+    *start = lstart;
+    *kount = lkount;
+}
+
+/**
+ * Look for the largest block of data for io which can be expressed in
+ * terms of start and count.
+ *
+ * @param arrlen
+ * @param arr_in
+ * @returns
+ */
+PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
+{
+    int i, j, n, numblks, numtimes, ii, numgaps;
+    PIO_Offset bsize, bsizeg, blklensum;
+    PIO_Offset del_arr[arrlen - 1];
+    PIO_Offset loc_arr[arrlen - 1];
+
+    numblks = 0;
+    numgaps = 0;
+    numtimes = 0;
+
+    for (i = 0; i < arrlen - 1; i++)
+    {
+        del_arr[i] = arr_in[i + 1] - arr_in[i];
+        if (del_arr[i] != 1)
+        {
+            numtimes++;
+            if ( i > 0 && del_arr[i - 1] > 1)
+                return(1);
+        }
+    }
+
+    numblks = numtimes + 1;
+    if (numtimes == 0)
+        n = numblks;
+    else
+        n = numtimes;
+
+    bsize = (PIO_Offset)arrlen;
+    if (numblks > 1)
+    {
+        PIO_Offset blk_len[numblks];
+        PIO_Offset gaps[numtimes];
+        if(numtimes > 0)
+        {
+            ii = 0;
+            for (i = 0; i < arrlen - 1; i++)
+                if (del_arr[i] > 1)
+                    gaps[ii++] = del_arr[i] - 1;
+            numgaps = ii;
+        }
+
+        j = 0;
+        for (i = 0; i < n; i++)
+            loc_arr[i] = 1;
+
+        for (i = 0; i < arrlen - 1; i++)
+            if(del_arr[i] != 1)
+                loc_arr[j++] = i;
+
+        blk_len[0] = loc_arr[0];
+        blklensum = blk_len[0];
+        for(i = 1; i < numblks - 1; i++)
+        {
+            blk_len[i] = loc_arr[i] - loc_arr[i - 1];
+            blklensum += blk_len[i];
+        }
+        blk_len[numblks - 1] = arrlen - blklensum;
+
+        bsize = lgcd_array(numblks, blk_len);
+        if (numgaps > 0)
+        {
+            bsizeg = lgcd_array(numgaps, gaps);
+            bsize = lgcd(bsize, bsizeg);
+        }
+        if(arr_in[0] > 0)
+            bsize = lgcd(bsize, arr_in[0]);
+    }
+    return bsize;
+}
+
+/**
+ * Compute start and count values for each io task.
+ *
+ * @param basetype
+ * @param ndims
+ * @param gdims
+ * @param num_io_procs
+ * @param myiorank
+ * @param start
+ * @param kount
+ */
+int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_procs,
+                      int myiorank, PIO_Offset *start, PIO_Offset *kount)
+{
+    int minbytes;
+    int maxbytes;
+    int minblocksize;
+    int basesize;
+    int use_io_procs;
+    int i;
+    long int p;
+    long int pgdims;
+    bool converged;
+    int iorank;
+    int ldims;
+    int tiorank;
+    int ioprocs;
+    int tioprocs;
+    int mystart[ndims], mykount[ndims];
+    long int pknt;
+    long int tpsize=0;
+
+    minbytes = blocksize - 256;
+    maxbytes  = blocksize + 256;
+
+    switch (basetype)
+    {
+    case PIO_INT:
+        basesize = sizeof(int);
+        break;
+    case PIO_REAL:
+        basesize = sizeof(float);
+        break;
+    case PIO_DOUBLE:
+        basesize = sizeof(double);
+        break;
+    default:
+        piodie("Invalid basetype ",__FILE__,__LINE__);
+        break;
+    }
+    minblocksize = minbytes / basesize;
+
+    pgdims = 1;
+    for (i = 0; i < ndims; i++)
+        pgdims *= (long int)gdims[i];
+    p = pgdims;
+    use_io_procs = max(1, min((int)((float)p / (float)minblocksize + 0.5), num_io_procs));
+    converged = 0;
+    for (i = 0; i < ndims; i++)
+    {
+        mystart[i] = 0;
+        mykount[i] = 0;
+    }
+
+    while (!converged)
+    {
+        for (iorank = 0; iorank < use_io_procs; iorank++)
+        {
+            for (i = 0; i < ndims; i++)
+            {
+                start[i] = 0;
+                kount[i] = gdims[i];
+            }
+            ldims = ndims - 1;
+            p = basesize;
+            for (i = ndims - 1; i >= 0; i--)
+            {
+                p = p * gdims[i];
+                if (p / use_io_procs > maxbytes)
+                {
+                    ldims = i;
+                    break;
+                }
+            }
+
+            if (gdims[ldims] < use_io_procs)
+            {
+                if (ldims > 0 && gdims[ldims-1] > use_io_procs)
+                    ldims--;
+                else
+                    use_io_procs -= (use_io_procs % gdims[ldims]);
+            }
+
+            ioprocs = use_io_procs;
+            tiorank = iorank;
+            for (i = 0; i <= ldims; i++)
+            {
+                if (gdims[i] >= ioprocs)
+                {
+                    computestartandcount(gdims[i], ioprocs, tiorank, start + i, kount + i);
+                    if (start[i] + kount[i] > gdims[i] + 1)
+                    {
+                        piodie("Start plus count exceeds dimension bound",__FILE__,__LINE__);
+                    }
+                }
+                else if(gdims[i] > 1)
+                {
+                    tioprocs = gdims[i];
+                    tiorank = (iorank * tioprocs) / ioprocs;
+                    computestartandcount(gdims[i], tioprocs, tiorank, start + i, kount + i);
+                    ioprocs = ioprocs / tioprocs;
+                    tiorank  = iorank % ioprocs;
+                }
+
+            }
+
+            if (myiorank == iorank)
+            {
+                for (i = 0; i < ndims; i++)
+                {
+                    mystart[i] = start[i];
+                    mykount[i] = kount[i];
+                }
+            }
+            pknt = 1;
+
+            for(i = 0; i < ndims; i++)
+                pknt *= kount[i];
+
+            tpsize += pknt;
+
+            if (tpsize == pgdims && use_io_procs == iorank + 1)
+            {
+                converged = true;
+                break;
+            }
+            else if(tpsize >= pgdims)
+            {
+                break;
+            }
+        }
+
+        if (!converged)
+        {
+            tpsize = 0;
+            use_io_procs--;
+        }
+    }
+
+    if (myiorank < use_io_procs)
+    {
+        for (i = 0; i < ndims; i++)
+        {
+            start[i] = mystart[i];
+            kount[i] = mykount[i];
+        }
+    }
+    else
+    {
+        for (i = 0; i < ndims; i++)
+        {
+            start[i] = 0;
+            kount[i] = 0;
+        }
+    }
+
+    return use_io_procs;
+}
diff --git a/src/externals/pio2/src/clib/pioc_support.c b/src/externals/pio2/src/clib/pioc_support.c
new file mode 100644
index 00000000000..8f45277a32f
--- /dev/null
+++ b/src/externals/pio2/src/clib/pioc_support.c
@@ -0,0 +1,2424 @@
+/** @file
+ * Support functions for the PIO library.
+ */
+#include 
+#if PIO_ENABLE_LOGGING
+#include 
+#include 
+#endif /* PIO_ENABLE_LOGGING */
+#include 
+#include 
+
+#include 
+
+#define VERSNO 2001
+
+/* Some logging constants. */
+#if PIO_ENABLE_LOGGING
+#define MAX_LOG_MSG 1024
+#define MAX_RANK_STR 12
+#define ERROR_PREFIX "ERROR: "
+#define NC_LEVEL_DIFF 3
+int pio_log_level = 0;
+int pio_log_ref_cnt = 0;
+int my_rank;
+FILE *LOG_FILE = NULL;
+#endif /* PIO_ENABLE_LOGGING */
+
+/**
+ * The PIO library maintains its own set of ncids. This is the next
+ * ncid number that will be assigned.
+ */
+extern int pio_next_ncid;
+
+/** The default error handler used when iosystem cannot be located. */
+extern int default_error_handler;
+
+/**
+ * Return a string description of an error code. If zero is passed,
+ * the errmsg will be "No error".
+ *
+ * @param pioerr the error code returned by a PIO function call.
+ * @param errmsg Pointer that will get the error message. The message
+ * will be PIO_MAX_NAME chars or less.
+ * @return 0 on success.
+ */
+int PIOc_strerror(int pioerr, char *errmsg)
+{
+    LOG((1, "PIOc_strerror pioerr = %d", pioerr));
+
+    /* Caller must provide this. */
+    pioassert(errmsg, "pointer to errmsg string must be provided", __FILE__, __LINE__);
+
+    /* System error? NetCDF and pNetCDF errors are always negative. */
+    if (pioerr > 0)
+    {
+        const char *cp = (const char *)strerror(pioerr);
+        if (cp)
+            strncpy(errmsg, cp, PIO_MAX_NAME);
+        else
+            strcpy(errmsg, "Unknown Error");
+    }
+    else if (pioerr == PIO_NOERR)
+        strcpy(errmsg, "No error");
+#if defined(_NETCDF)
+    else if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR)     /* NetCDF error? */
+        strncpy(errmsg, nc_strerror(pioerr), NC_MAX_NAME);
+#endif /* endif defined(_NETCDF) */
+#if defined(_PNETCDF)
+    else if (pioerr > PIO_FIRST_ERROR_CODE)     /* pNetCDF error? */
+        strncpy(errmsg, ncmpi_strerror(pioerr), NC_MAX_NAME);
+#endif /* defined( _PNETCDF) */
+    else
+        /* Handle PIO errors. */
+        switch(pioerr)
+        {
+        case PIO_EBADIOTYPE:
+            strcpy(errmsg, "Bad IO type");
+            break;
+        default:
+            strcpy(errmsg, "Unknown Error: Unrecognized error code");
+        }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set the logging level if PIO was built with
+ * PIO_ENABLE_LOGGING. Set to -1 for nothing, 0 for errors only, 1 for
+ * important logging, and so on. Log levels below 1 are only printed
+ * on the io/component root.
+ *
+ * A log file is also produced for each task. The file is called
+ * pio_log_X.txt, where X is the (0-based) task number.
+ *
+ * If the library is not built with logging, this function does
+ * nothing.
+ *
+ * @param level the logging level, 0 for errors only, 5 for max
+ * verbosity.
+ * @returns 0 on success, error code otherwise.
+ */
+int PIOc_set_log_level(int level)
+{
+    int ret;
+
+#if PIO_ENABLE_LOGGING
+    /* Set the log level. */
+    pio_log_level = level;
+
+#if NETCDF_C_LOGGING_ENABLED
+    /* If netcdf logging is available turn it on starting at level = 4. */
+    if (level > NC_LEVEL_DIFF)
+        if ((ret = nc_set_log_level(level - NC_LEVEL_DIFF)))
+            return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+#endif /* NETCDF_C_LOGGING_ENABLED */
+#endif /* PIO_ENABLE_LOGGING */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Initialize logging.  Open log file, if not opened yet, or increment
+ * ref count if already open.
+ */
+void pio_init_logging(void)
+{
+#if PIO_ENABLE_LOGGING
+    char log_filename[NC_MAX_NAME];
+
+    if (!LOG_FILE)
+    {
+        /* Create a filename with the rank in it. */
+        MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+        sprintf(log_filename, "pio_log_%d.txt", my_rank);
+
+        /* Open a file for this rank to log messages. */
+        LOG_FILE = fopen(log_filename, "w");
+
+        pio_log_ref_cnt = 1;
+    }
+    else
+    {
+        pio_log_ref_cnt++;
+    }
+#endif /* PIO_ENABLE_LOGGING */
+}
+
+/**
+ * Finalize logging - close log files, if open.
+ */
+void pio_finalize_logging(void )
+{
+#if PIO_ENABLE_LOGGING
+    pio_log_ref_cnt -= 1;
+    if (LOG_FILE)
+    {
+        if (pio_log_ref_cnt == 0)
+        {
+            fclose(LOG_FILE);
+            LOG_FILE = NULL;
+        }
+        else
+            LOG((2, "pio_finalize_logging, postpone close, ref_cnt = %d",
+                 pio_log_ref_cnt));
+    }
+#endif /* PIO_ENABLE_LOGGING */
+}
+
+#if PIO_ENABLE_LOGGING
+/**
+ * This function prints out a message, if the severity of the message
+ * is lower than the global pio_log_level. To use it, do something
+ * like this:
+ *
+ * pio_log(0, "this computer will explode in %d seconds", i);
+ *
+ * After the first arg (the severity), use the rest like a normal
+ * printf statement. Output will appear on stdout.
+ * This function is heavily based on the function in section 15.5 of
+ * the C FAQ.
+ *
+ * In code this functions should be wrapped in the LOG(()) macro.
+ *
+ * @param severity the severity of the message, 0 for error messages,
+ * then increasing levels of verbosity.
+ * @param fmt the format string.
+ * @param ... the arguments used in format string.
+ */
+void pio_log(int severity, const char *fmt, ...)
+{
+    va_list argp;
+    int t;
+    int rem_len = MAX_LOG_MSG;
+    char msg[MAX_LOG_MSG];
+    char *ptr = msg;
+    char rank_str[MAX_RANK_STR];
+
+    /* If the severity is greater than the log level, we don't print
+       this message. */
+    if (severity > pio_log_level)
+        return;
+
+    /* If the severity is 0, only print on rank 0. */
+    if (severity < 1 && my_rank != 0)
+        return;
+
+    /* If the severity is zero, this is an error. Otherwise insert that
+       many tabs before the message. */
+    if (!severity)
+    {
+        strncpy(ptr, ERROR_PREFIX, (rem_len > 0) ? rem_len : 0);
+        ptr += strlen(ERROR_PREFIX);
+        rem_len -= strlen(ERROR_PREFIX);
+    }
+    for (t = 0; t < severity; t++)
+    {
+        strncpy(ptr++, "\t", (rem_len > 0) ? rem_len : 0);
+        rem_len--;
+    }
+
+    /* Show the rank. */
+    snprintf(rank_str, MAX_RANK_STR, "%d ", my_rank);
+    strncpy(ptr, rank_str, (rem_len > 0) ? rem_len : 0);
+    ptr += strlen(rank_str);
+    rem_len -= strlen(rank_str);
+
+    /* Print out the variable list of args with vprintf. */
+    va_start(argp, fmt);
+    vsnprintf(ptr, ((rem_len > 0) ? rem_len : 0), fmt, argp);
+    va_end(argp);
+
+    /* Put on a final linefeed. */
+    ptr = msg + strlen(msg);
+    rem_len = MAX_LOG_MSG - strlen(msg);
+    strncpy(ptr, "\n\0", (rem_len > 0) ? rem_len : 0);
+
+    /* Send message to stdout. */
+    fprintf(stdout, "%s", msg);
+
+    /* Send message to log file. */
+    if (LOG_FILE)
+        fprintf(LOG_FILE, "%s", msg);
+
+    /* Ensure an immediate flush of stdout. */
+    fflush(stdout);
+    if (LOG_FILE)
+        fflush(LOG_FILE);
+}
+#endif /* PIO_ENABLE_LOGGING */
+
+/**
+ * Obtain a backtrace and print it to stderr. This is appended to the
+ * text decomposition file.
+ *
+ * Note from Jim:
+ *
+ * The stack trace can be used to identify the usage in
+ * the model code of the particular decomposition in question and so
+ * if using the pio performance tool leads to tuning that could be
+ * applied in the model you know more or less where to do it.
+ *
+ * It's also useful if you have a model bug - then you have 20 or so
+ * decomp files and you need to identify the one that was problematic.
+ * So it's used as an add to the developer and not used at all by any
+ * automated process or tools.
+ *
+ * @param fp file pointer to send output to
+ */
+void print_trace(FILE *fp)
+{
+    void *array[10];
+    size_t size;
+    char **strings;
+    size_t i;
+
+    /* Note that this won't actually work. */
+    if (fp == NULL)
+        fp = stderr;
+
+    size = backtrace(array, 10);
+    strings = backtrace_symbols(array, size);
+
+    fprintf(fp,"Obtained %zd stack frames.\n", size);
+
+    for (i = 0; i < size; i++)
+        fprintf(fp,"%s\n", strings[i]);
+
+    free(strings);
+}
+
+/**
+ * Exit due to lack of memory.
+ *
+ * @param ios the iosystem description struct
+ * @param req amount of memory that was being requested
+ * @param fname name of code file where error occured
+ * @param line the line of code where the error occurred.
+ */
+void piomemerror(iosystem_desc_t *ios, size_t req, char *fname, int line)
+{
+    char msg[80];
+    sprintf(msg, "out of memory requesting: %ld", req);
+    cn_buffer_report(ios, false);
+    piodie(msg, fname, line);
+}
+
+/**
+ * Abort program and call MPI_Abort().
+ *
+ * @param msg an error message
+ * @param fname name of code file where error occured
+ * @param line the line of code where the error occurred.
+ */
+void piodie(const char *msg, const char *fname, int line)
+{
+    fprintf(stderr,"Abort with message %s in file %s at line %d\n",
+            msg ? msg : "_", fname ? fname : "_", line);
+
+    print_trace(stderr);
+#ifdef MPI_SERIAL
+    abort();
+#else
+    MPI_Abort(MPI_COMM_WORLD, -1);
+#endif
+}
+
+/**
+ * Perform an assert. Note that this function does nothing if NDEBUG
+ * is defined.
+ *
+ * @param expression the expression to be evaluated
+ * @param msg an error message
+ * @param fname name of code file where error occured
+ * @param line the line of code where the error occurred.
+ */
+void pioassert(_Bool expression, const char *msg, const char *fname, int line)
+{
+#ifndef NDEBUG
+    if (!expression)
+        piodie(msg, fname, line);
+#endif
+}
+
+/**
+ * Handle MPI errors. An error message is sent to stderr, then the
+ * check_netcdf() function is called with PIO_EIO.
+ *
+ * @param file pointer to the file_desc_t info. Ignored if NULL.
+ * @param mpierr the MPI return code to handle
+ * @param filename the name of the code file where error occured.
+ * @param line the line of code where error occured.
+ * @return PIO_NOERR for no error, otherwise PIO_EIO.
+ */
+int check_mpi(file_desc_t *file, int mpierr, const char *filename,
+              int line)
+{
+    return check_mpi2(NULL, file, mpierr, filename, line);
+}
+
+/**
+ * Handle MPI errors. An error message is sent to stderr, then the
+ * check_netcdf() function is called with PIO_EIO. This version of the
+ * function accepts an ios parameter, for the (rare) occasions where
+ * we have an ios but not a file.
+ *
+ * @param ios pointer to the iosystem_info_t. May be NULL.
+ * @param file pointer to the file_desc_t info. May be NULL.
+ * @param mpierr the MPI return code to handle
+ * @param filename the name of the code file where error occured.
+ * @param line the line of code where error occured.
+ * @return PIO_NOERR for no error, otherwise PIO_EIO.
+ */
+int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr,
+               const char *filename, int line)
+{
+    if (mpierr)
+    {
+        char errstring[MPI_MAX_ERROR_STRING];
+        int errstrlen;
+
+        /* If we can get an error string from MPI, print it to stderr. */
+        if (!MPI_Error_string(mpierr, errstring, &errstrlen))
+            fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n",
+                    errstring, filename ? filename : "_", line);
+
+        /* Handle all MPI errors as PIO_EIO. */
+        return pio_err(ios, file, PIO_EIO, filename, line);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Check the result of a netCDF API call.
+ *
+ * @param file pointer to the PIO structure describing this
+ * file. Ignored if NULL.
+ * @param status the return value from the netCDF call.
+ * @param fname the name of the code file.
+ * @param line the line number of the netCDF call in the code.
+ * @return the error code
+ */
+int check_netcdf(file_desc_t *file, int status, const char *fname, int line)
+{
+    return check_netcdf2(NULL, file, status, fname, line);
+}
+
+/**
+ * Check the result of a netCDF API call. This is the same as
+ * check_netcdf() except for the extra iosystem_desc_t pointer, which
+ * is used to determine error handling when there is no file_desc_t
+ * pointer.
+ *
+ * @param ios pointer to the iosystem description struct. Ignored if NULL.
+ * @param file pointer to the PIO structure describing this file. Ignored if NULL.
+ * @param status the return value from the netCDF call.
+ * @param fname the name of the code file.
+ * @param line the line number of the netCDF call in the code.
+ * @return the error code
+ */
+int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
+                  const char *fname, int line)
+{
+    int eh = default_error_handler; /* Error handler that will be used. */
+    char errmsg[PIO_MAX_NAME + 1];  /* Error message. */
+
+    /* User must provide this. */
+    pioassert(fname, "code file name must be provided", __FILE__, __LINE__);
+
+    /* No harm, no foul. */
+    if (status == PIO_NOERR)
+        return PIO_NOERR;
+
+    LOG((1, "check_netcdf2 status = %d fname = %s line = %d", status, fname, line));
+
+    /* Pick an error handler. */
+    if (ios)
+        eh = ios->error_handler;
+    if (file)
+        eh = file->iosystem->error_handler;
+    pioassert(eh == PIO_INTERNAL_ERROR || eh == PIO_BCAST_ERROR || eh == PIO_RETURN_ERROR,
+              "invalid error handler", __FILE__, __LINE__);
+    LOG((2, "check_netcdf2 chose error handler = %d", eh));
+
+    /* Get an error message. */
+    if (eh != PIO_BCAST_ERROR && !PIOc_strerror(status, errmsg))
+    {
+        fprintf(stderr, "%s\n", errmsg);
+        LOG((1, "check_netcdf2 errmsg = %s", errmsg));
+    }
+
+    /* Decide what to do based on the error handler. */
+    if (eh == PIO_INTERNAL_ERROR)
+        piodie(errmsg, fname, line);        /* Die! */
+    else if (eh == PIO_BCAST_ERROR)
+    {
+	if (ios)
+	    MPI_Bcast(&status, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	else if (file)
+	    MPI_Bcast(&status, 1, MPI_INT, file->iosystem->ioroot, file->iosystem->my_comm);
+    }
+
+    /* For PIO_RETURN_ERROR, just return the error. */
+    return status;
+}
+
+/**
+ * Handle an error in PIO. This will consult the error handler
+ * settings and either call MPI_Abort() or return an error code.
+ *
+ * The error hanlder has three settings:
+ *
+ * Errors cause abort: PIO_INTERNAL_ERROR.
+ *
+ * Error codes are broadcast to all tasks: PIO_BCAST_ERROR.
+ *
+ * Errors are returned to caller with no internal action:
+ * PIO_RETURN_ERROR.
+ *
+ * @param ios pointer to the IO system info. Ignored if NULL.
+ * @param file pointer to the file description data. Ignored if
+ * NULL. If provided file->iosystem is used as ios pointer.
+ * @param err_num the error code
+ * @param fname name of code file where error occured.
+ * @param line the line of code where the error occurred.
+ * @returns err_num if abort is not called.
+ */
+int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
+            int line)
+{
+    char err_msg[PIO_MAX_NAME + 1];
+    int err_handler = default_error_handler; /* Default error handler. */
+    int ret;
+
+    /* User must provide this. */
+    pioassert(fname, "file name must be provided", __FILE__, __LINE__);
+
+    /* No harm, no foul. */
+    if (err_num == PIO_NOERR)
+        return PIO_NOERR;
+
+    /* Get the error message. */
+    if ((ret = PIOc_strerror(err_num, err_msg)))
+        return ret;
+
+    /* If logging is in use, log an error message. */
+    LOG((0, "%s err_num = %d fname = %s line = %d", err_msg, err_num, fname ? fname : '\0', line));
+
+    /* What error handler should we use? */
+    if (file)
+        err_handler = file->iosystem->error_handler;
+    else if (ios)
+        err_handler = ios->error_handler;
+
+    LOG((2, "pio_err chose error handler = %d", err_handler));
+
+    /* Should we abort? */
+    if (err_handler == PIO_INTERNAL_ERROR)
+    {
+        /* For debugging only, this will print a traceback of the call tree.  */
+        print_trace(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    /* What should we do here??? */
+    if (err_handler == PIO_BCAST_ERROR)
+    {
+        /* ??? */
+    }
+
+    /* If abort was not called, we'll get here. */
+    return err_num;
+}
+
+/**
+ * Allocate an region.
+ *
+ * ndims the number of dimensions for the data in this region.
+ * @returns a pointer to the newly allocated io_region struct.
+ */
+io_region *alloc_region(int ndims)
+{
+    io_region *region;
+
+    /* Allocate memory for the io_region struct. */
+    if (!(region = bget(sizeof(io_region))))
+        return NULL;
+
+    /* Allocate memory for the array of start indicies. */
+    if (!(region->start = bget(ndims * sizeof(PIO_Offset))))
+    {
+        brel(region);
+        return NULL;
+    }
+
+    /* Allocate memory for the array of counts. */
+    if (!(region->count = bget(ndims * sizeof(PIO_Offset))))
+    {
+        brel(region);
+        brel(region->start);
+        return NULL;
+    }
+
+    region->loffset = 0;
+    region->next = NULL;
+
+    /* Initialize start and count arrays to zero. */
+    for (int i = 0; i < ndims; i++)
+    {
+        region->start[i] = 0;
+        region->count[i] = 0;
+    }
+
+    return region;
+}
+
+/**
+ * Find the MPI type for a PIO type.
+ *
+ * @param pio_type a PIO type, PIO_INT, PIO_FLOAT, etc.
+ * @param mpi_type a pointer to MPI_Datatype that will get the MPI
+ * type that coresponds to the PIO type.
+ * @returns 0 for success, error code otherwise.
+ */
+int find_mpi_type(int pio_type, MPI_Datatype *mpi_type)
+{
+    /* Check input. */
+    pioassert(mpi_type, "invalid input", __FILE__, __LINE__);
+
+    /* Decide on the base type. */
+    switch(pio_type)
+    {
+    case PIO_BYTE:
+        *mpi_type = MPI_BYTE;
+        break;
+    case PIO_CHAR:
+        *mpi_type = MPI_CHAR;
+        break;
+    case PIO_SHORT:
+        *mpi_type = MPI_SHORT;
+        break;
+    case PIO_INT:
+        *mpi_type = MPI_INT;
+        break;
+    case PIO_FLOAT:
+        *mpi_type = MPI_FLOAT;
+        break;
+    case PIO_DOUBLE:
+        *mpi_type = MPI_DOUBLE;
+        break;
+#ifdef _NETCDF4
+    case PIO_UBYTE:
+        *mpi_type = MPI_UNSIGNED_CHAR;
+        break;
+    case PIO_USHORT:
+        *mpi_type = MPI_UNSIGNED_SHORT;
+        break;
+    case PIO_UINT:
+        *mpi_type = MPI_UNSIGNED;
+        break;
+    case PIO_INT64:
+        *mpi_type = MPI_LONG_LONG;
+        break;
+    case PIO_UINT64:
+        *mpi_type = MPI_UNSIGNED_LONG_LONG;
+        break;
+    case PIO_STRING:
+        *mpi_type = MPI_CHAR;
+        break;
+#endif /* _NETCDF4 */
+    default:
+        return PIO_EBADTYPE;
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Allocate space for an IO description struct.
+ *
+ * @param ios pointer to the IO system info.
+ * @param piotype the PIO data type (ex. PIO_FLOAT, PIO_INT, etc.).
+ * @param ndims the number of dimensions.
+ * @iodesc pointer that gets a pointer to the newly allocated
+ * io_desc_t or NULL if allocation failed.
+ * @returns 0 for success, error code otherwise.
+ */
+int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims,
+                  io_desc_t **iodesc)
+{
+    MPI_Datatype mpi_type;
+    int ret;
+
+    /* Check input. */
+    pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
+
+    /* Allocate space for the io_desc_t struct. */
+    if (!(*iodesc = calloc(1, sizeof(io_desc_t))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Get the MPI type corresponding with the PIO type. */
+    if ((ret = find_mpi_type(piotype, &mpi_type)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Decide on the base type. */
+    (*iodesc)->basetype = mpi_type;
+
+    /* Initialize some values in the struct. */
+    (*iodesc)->maxregions = 1;
+    (*iodesc)->ioid = -1;
+    (*iodesc)->ndims = ndims;
+    (*iodesc)->firstregion = alloc_region(ndims);
+
+    /* Set the swap memory settings to defaults. */
+    (*iodesc)->rearr_opts = ios->rearr_opts;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Free a region list.
+ *
+ * top a pointer to the start of the list to free.
+ */
+void free_region_list(io_region *top)
+{
+    io_region *ptr, *tptr;
+
+    ptr = top;
+    while (ptr)
+    {
+        if (ptr->start)
+            brel(ptr->start);
+        if (ptr->count)
+            brel(ptr->count);
+        tptr = ptr;
+        ptr = ptr->next;
+        brel(tptr);
+    }
+}
+
+/**
+ * Free a decomposition map.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition map to free.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_freedecomp(int iosysid, int ioid)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_FREEDECOMP; /* Message for async notification. */
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_freedecomp iosysid = %d ioid = %d", iosysid, ioid));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Free the map. */
+    free(iodesc->map);
+
+    /* Free the dimlens. */
+    free(iodesc->dimlen);
+
+    if (iodesc->rfrom)
+        free(iodesc->rfrom);
+
+    if (iodesc->rtype)
+    {
+        for (int i = 0; i < iodesc->nrecvs; i++)
+            if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
+                if ((mpierr = MPI_Type_free(iodesc->rtype + i)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+        free(iodesc->rtype);
+    }
+
+    if (iodesc->stype)
+    {
+        for (int i = 0; i < iodesc->num_stypes; i++)
+            if (iodesc->stype[i] != PIO_DATATYPE_NULL)
+                if ((mpierr = MPI_Type_free(iodesc->stype + i)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+        iodesc->num_stypes = 0;
+        free(iodesc->stype);
+    }
+
+    if (iodesc->scount)
+        free(iodesc->scount);
+
+    if (iodesc->rcount)
+        free(iodesc->rcount);
+
+    if (iodesc->sindex)
+        free(iodesc->sindex);
+
+    if (iodesc->rindex)
+        free(iodesc->rindex);
+
+    if (iodesc->firstregion)
+        free_region_list(iodesc->firstregion);
+
+    if (iodesc->fillregion)
+        free_region_list(iodesc->fillregion);
+
+    if (iodesc->rearranger == PIO_REARR_SUBSET)
+        if ((mpierr = MPI_Comm_free(&iodesc->subset_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    return pio_delete_iodesc_from_list(ioid);
+}
+
+/**
+ * Read a decomposition map from a file. The decomp file is only read
+ * by task 0 in the communicator.
+ *
+ * @param file the filename
+ * @param ndims pointer to an int with the number of dims.
+ * @param gdims pointer to an array of dimension ids.
+ * @param fmaplen
+ * @param map
+ * @param comm
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
+                 PIO_Offset **map, MPI_Comm comm)
+{
+    int npes, myrank;
+    int rnpes, rversno;
+    int j;
+    int *tdims;
+    PIO_Offset *tmap;
+    MPI_Status status;
+    PIO_Offset maplen;
+    int mpierr; /* Return code for MPI calls. */
+
+    /* Check inputs. */
+    if (!file || !ndims || !gdims || !fmaplen || !map)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Comm_size(comm, &npes)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &myrank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    if (myrank == 0)
+    {
+        FILE *fp = fopen(file, "r");
+        if (!fp)
+            pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+        fscanf(fp,"version %d npes %d ndims %d\n",&rversno, &rnpes, ndims);
+
+        if (rversno != VERSNO)
+            return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+        if (rnpes < 1 || rnpes > npes)
+            return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+        if ((mpierr = MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(ndims, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if (!(tdims = calloc(*ndims, sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int i = 0; i < *ndims; i++)
+            fscanf(fp,"%d ", tdims + i);
+
+        if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        for (int i = 0; i < rnpes; i++)
+        {
+            fscanf(fp, "%d %lld", &j, &maplen);
+            if (j != i)  // Not sure how this could be possible
+                return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+            if (!(tmap = malloc(maplen * sizeof(PIO_Offset))))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+            for (j = 0; j < maplen; j++)
+                fscanf(fp, "%lld ", tmap+j);
+
+            if (i > 0)
+            {
+                if ((mpierr = MPI_Send(&maplen, 1, PIO_OFFSET, i, i + npes, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmap, maplen, PIO_OFFSET, i, i, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                free(tmap);
+            }
+            else
+            {
+                *map = tmap;
+                *fmaplen = maplen;
+            }
+        }
+        fclose(fp);
+    }
+    else
+    {
+        if ((mpierr = MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(ndims, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if (!(tdims = calloc(*ndims, sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        if (myrank < rnpes)
+        {
+            if ((mpierr = MPI_Recv(&maplen, 1, PIO_OFFSET, 0, myrank + npes, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if (!(tmap = malloc(maplen * sizeof(PIO_Offset))))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+            if ((mpierr = MPI_Recv(tmap, maplen, PIO_OFFSET, 0, myrank, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            *map = tmap;
+        }
+        else
+        {
+            tmap = NULL;
+            maplen = 0;
+        }
+        *fmaplen = maplen;
+    }
+    *gdims = tdims;
+    return PIO_NOERR;
+}
+
+/**
+ * Read a decomposition map from file.
+ *
+ * @param file the filename
+ * @param ndims pointer to the number of dimensions
+ * @param gdims pointer to an array of dimension ids
+ * @param maplen pointer to the length of the map
+ * @param map pointer to the map array
+ * @param f90_comm
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset *maplen,
+                          PIO_Offset **map, int f90_comm)
+{
+    return PIOc_readmap(file, ndims, gdims, maplen, map, MPI_Comm_f2c(f90_comm));
+}
+
+/**
+ * Write the decomposition map to a file using netCDF, everyones
+ * favorite data format.
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the filename to be used.
+ * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE.
+ * @param ioid the ID of the IO description.
+ * @param comm an MPI communicator.
+ * @param title optial title attribute for the file. Must be less than
+ * NC_MAX_NAME + 1 if provided. Ignored if NULL.
+ * @param history optial history attribute for the file. Must be less
+ * than NC_MAX_NAME + 1 if provided. Ignored if NULL.
+ * @param fortran_order set to non-zero if fortran array ordering is
+ * used, or to zero if C array ordering is used.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
+                         MPI_Comm comm, char *title, char *history, int fortran_order)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int npes;   /* Size of this communicator. */
+    int myrank; /* Rank of this task. */
+    int mpierr;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_write_nc_decomp filename = %s iosysid = %d ioid = %d", filename,
+         iosysid, ioid));
+
+    /* Get the IO desc, which describes the decomposition. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Get the communicator size and task rank. */
+    if ((mpierr = MPI_Comm_size(comm, &npes)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &myrank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "npes = %d myrank = %d", npes, myrank));
+
+    /* Allocate memory for the nmaplen. On task 0, this will contain
+     * the length of the map on each task, for all tasks. */
+    int task_maplen[npes];
+
+    /* Gather maplens from all tasks and fill the task_maplen array on
+     * all tasks. */
+    if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT, comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* We will need to know the maximum maplen used for any task. */
+    int max_maplen;
+
+    /* Find the max maxplen. */
+    if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX, comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((3, "max_maplen = %d", max_maplen));
+
+    /* 2D array that, on task 0, will hold all the map information for
+     * all tasks. */
+    int full_map[npes][max_maplen];
+
+    /* Fill local array with my map. Use the fill value for unused */
+    /* elements at the end if max_maplen is longer than maplen. Also
+     * subtract 1 because the iodesc->map is 1-based. */
+    int my_map[max_maplen];
+    for (int e = 0; e < max_maplen; e++)
+    {
+        my_map[e] = e < iodesc->maplen ? iodesc->map[e] - 1 : NC_FILL_INT;
+        LOG((3, "my_map[%d] = %d", e, my_map[e]));
+    }
+
+    /* Gather my_map from all tasks and fill the full_map array. */
+    if ((mpierr = MPI_Allgather(&my_map, max_maplen, MPI_INT, full_map, max_maplen,
+                             MPI_INT, comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    for (int p = 0; p < npes; p++)
+        for (int e = 0; e < max_maplen; e++)
+            LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p][e]));
+
+    /* Write the netCDF decomp file. */
+    if ((ret = pioc_write_nc_decomp_int(iosysid, filename, cmode, iodesc->ndims, iodesc->dimlen, npes,
+                                        task_maplen, (int *)full_map, title, history, fortran_order)))
+        return ret;
+
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read the decomposition map from a netCDF decomp file produced by
+ * PIOc_write_nc_decomp().
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the name of the decomp file.
+ * @param ioid pointer that will get the newly-assigned ID of the IO
+ * description. The ioid is needed to later free the decomposition.
+ * @param comm an MPI communicator.
+ * @param pio_type the PIO type to be used as the type for the data.
+ * @param title pointer that will get optial title attribute for the
+ * file. Will be less than NC_MAX_NAME + 1 if provided. Ignored if
+ * NULL.
+ * @param history pointer that will get optial history attribute for
+ * the file. Will be less than NC_MAX_NAME + 1 if provided. Ignored if
+ * NULL.
+ * @param fortran_order pointer that gets set to 1 if fortran array
+ * ordering is used, or to zero if C array ordering is used.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm comm,
+                        int pio_type, char *title, char *history, int *fortran_order)
+{
+    iosystem_desc_t *ios; /* Pointer to the IO system info. */
+    int ndims;            /* The number of data dims (except unlim). */
+    int max_maplen;       /* The max maplen of any task. */
+    int *global_dimlen;   /* An array with sizes of global dimensions. */
+    int *task_maplen;     /* A map of one tasks mapping to global data. */
+    int *full_map;        /* A map with the task maps of every task. */
+    int num_tasks_decomp; /* The number of tasks for this decomp. */
+    int size;             /* Size of comm. */
+    int my_rank;          /* Task rank in comm. */
+    MPI_Datatype mpi_type;             /* Will be used as the basetype in iodesc. */
+    int mpi_type_int;                  /* int version of mpi_type. */
+    char source_in[PIO_MAX_NAME + 1];  /* Text metadata in decomp file. */
+    char version_in[PIO_MAX_NAME + 1]; /* Text metadata in decomp file. */
+    int mpierr;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename || !ioidp)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_read_nc_decomp filename = %s iosysid = %d pio_type = %d",
+         filename, iosysid, pio_type));
+
+    /* Get the MPI type. We need it as an int. */
+    if ((ret = find_mpi_type(pio_type, &mpi_type)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    mpi_type_int = mpi_type;
+    LOG((2, "mpi_type = %d mpi_type_int = %d", mpi_type, mpi_type_int));
+
+    /* Get the communicator size and task rank. */
+    if ((mpierr = MPI_Comm_size(comm, &size)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &my_rank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "size = %d my_rank = %d", size, my_rank));
+
+    /* Read the file. This allocates three arrays that we have to
+     * free. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, filename, &ndims, &global_dimlen, &num_tasks_decomp,
+                                       &task_maplen, &max_maplen, &full_map, title, history,
+                                       source_in, version_in, fortran_order)))
+        return ret;
+    LOG((2, "ndims = %d num_tasks_decomp = %d max_maplen = %d", ndims, num_tasks_decomp,
+         max_maplen));
+
+    /* If the size does not match the number of tasks in the decomp,
+     * that's an error. */
+    if (size != num_tasks_decomp)
+        ret = PIO_EINVAL;
+
+    /* Now initialize the iodesc on each task for this decomposition. */
+    if (!ret)
+    {
+        PIO_Offset compmap[task_maplen[my_rank]];
+
+        /* Copy array into PIO_Offset array. Make it 1 based. */
+        for (int e = 0; e < task_maplen[my_rank]; e++)
+            compmap[e] = full_map[my_rank * max_maplen + e] + 1;
+
+        /* Initialize the decomposition. */
+        ret = PIOc_InitDecomp(iosysid, pio_type, ndims, global_dimlen, task_maplen[my_rank],
+                              compmap, ioidp, NULL, NULL, NULL);
+    }
+
+    /* Free resources. */
+    free(global_dimlen);
+    free(task_maplen);
+    free(full_map);
+
+    return ret;
+}
+
+/* Write the decomp information in netCDF. This is an internal
+ * function.
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the name the decomp file will have.
+ * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE.
+ * @param ndims number of dims in the data being described.
+ * @param global_dimlen an array, of size ndims, with the size of the
+ * global array in each dimension.
+ * @param num_tasks the number of tasks the data are decomposed over.
+ * @param task_maplen array of size num_tasks with the length of the
+ * map for each task.
+ * @param map pointer to a 2D array of size [num_tasks][max_maplen]
+ * with the 0-based mapping from local to global array elements.
+ * @param title null-terminated string that will be written as an
+ * attribute. If provided, length must be < PIO_MAX_NAME + 1. Ignored
+ * if NULL.
+ * @param history null-terminated string that will be written as an
+ * attribute. If provided, length must be < PIO_MAX_NAME + 1. Ignored
+ * if NULL.
+ * @param fortran_order set to non-zero if using fortran array
+ * ordering, 0 for C array ordering.
+ * @returns 0 for success, error code otherwise.
+ */
+int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int ndims,
+                             int *global_dimlen, int num_tasks, int *task_maplen, int *map,
+                             const char *title, const char *history, int fortran_order)
+{
+    iosystem_desc_t *ios;
+    int max_maplen = 0;
+    int ncid;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename || !global_dimlen || !task_maplen)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    if (title)
+        if (strlen(title) > PIO_MAX_NAME)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    if (history)
+        if (strlen(history) > PIO_MAX_NAME)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((2, "pioc_write_nc_decomp_int iosysid = %d filename = %s ndims = %d num_tasks = %d",
+         iosysid, filename, ndims, num_tasks));
+
+    /* Find the maximum maplen. */
+    for (int t = 0; t < num_tasks; t++)
+        if (task_maplen[t] > max_maplen)
+            max_maplen = task_maplen[t];
+    LOG((3, "max_maplen = %d", max_maplen));
+
+    /* Create the netCDF decomp file. */
+    if ((ret = PIOc_create(iosysid, filename, cmode | NC_WRITE, &ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with the version of this file. */
+    char version[PIO_MAX_NAME + 1];
+    sprintf(version, "%d.%d.%d", PIO_VERSION_MAJOR, PIO_VERSION_MINOR, PIO_VERSION_PATCH);
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_VERSION_ATT_NAME,
+                                 strlen(version) + 1, version)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with the max map len. */
+    if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME,
+                                PIO_INT, 1, &max_maplen)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write title attribute, if the user provided one. */
+    if (title)
+        if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_TITLE_ATT_NAME,
+                                     strlen(title) + 1, title)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write history attribute, if the user provided one. */
+    if (history)
+        if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_HISTORY_ATT_NAME,
+                                     strlen(history) + 1, history)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write a source attribute. */
+    char source[] = "Decomposition file produced by PIO library.";
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_SOURCE_ATT_NAME,
+                                 strlen(source) + 1, source)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with array ordering (C or Fortran). */
+    char c_order_str[] = DECOMP_C_ORDER_STR;
+    char fortran_order_str[] = DECOMP_FORTRAN_ORDER_STR;
+    char *my_order_str = fortran_order ? fortran_order_str : c_order_str;
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_ORDER_ATT_NAME,
+                                 strlen(my_order_str) + 1, my_order_str)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with the stack trace. This can be helpful
+     * for debugging. */
+    #define MAX_BACKTRACE 10
+    void *bt[MAX_BACKTRACE];
+    size_t bt_size;
+    char **bt_strings;
+    bt_size = backtrace(bt, MAX_BACKTRACE);
+    bt_strings = backtrace_symbols(bt, bt_size);
+
+    /* Find the max size. */
+    int max_bt_size = 0;
+    for (int b = 0; b < bt_size; b++)
+        if (strlen(bt_strings[b]) > max_bt_size)
+            max_bt_size = strlen(bt_strings[b]);
+    if (max_bt_size > NC_MAX_NAME)
+        max_bt_size = NC_MAX_NAME;
+
+    /* Copy the backtrace into one long string. */
+    char full_bt[max_bt_size * bt_size + bt_size + 1];
+    full_bt[0] = '\0';
+    for (int b = 0; b < bt_size; b++)
+    {
+        strncat(full_bt, bt_strings[b], max_bt_size);
+        strcat(full_bt, "\n");
+    }
+    free(bt_strings);
+    printf("full_bt = %s", full_bt);
+
+    /* Write the stack trace as an attribute. */
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_BACKTRACE_ATT_NAME,
+                                 strlen(full_bt) + 1, full_bt)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* We need a dimension for the dimensions in the data. (Example:
+     * for 4D data we will need to store 4 dimension IDs.) */
+    int dim_dimid;
+    if ((ret = PIOc_def_dim(ncid, DECOMP_DIM_DIM, ndims, &dim_dimid)))
+        return ret;
+
+    /* We need a dimension for tasks. If we have 4 tasks, we need to
+     * store an array of length 4 with the size of the local array on
+     * each task. */
+    int task_dimid;
+    if ((ret = PIOc_def_dim(ncid, DECOMP_TASK_DIM_NAME, num_tasks, &task_dimid)))
+        return ret;
+
+    /* We need a dimension for the map. It's length may vary, we will
+     * use the max_maplen for the dimension size. */
+    int mapelem_dimid;
+    if ((ret = PIOc_def_dim(ncid, DECOMP_MAPELEM_DIM_NAME, max_maplen,
+                            &mapelem_dimid)))
+        return ret;
+
+    /* Define a var to hold the global size of the array for each
+     * dimension. */
+    int gsize_varid;
+    if ((ret = PIOc_def_var(ncid, DECOMP_GLOBAL_SIZE_VAR_NAME, NC_INT, 1,
+                            &dim_dimid, &gsize_varid)))
+        return ret;
+
+    /* Define a var to hold the length of the local array on each task. */
+    int maplen_varid;
+    if ((ret = PIOc_def_var(ncid, DECOMP_MAPLEN_VAR_NAME, NC_INT, 1, &task_dimid,
+                            &maplen_varid)))
+        return ret;
+
+    /* Define a 2D var to hold the length of the local array on each task. */
+    int map_varid;
+    int map_dimids[2] = {task_dimid, mapelem_dimid};
+    if ((ret = PIOc_def_var(ncid, DECOMP_MAP_VAR_NAME, NC_INT, 2, map_dimids,
+                            &map_varid)))
+        return ret;
+
+    /* End define mode, to write data. */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Write the global dimension sizes. */
+    if ((PIOc_put_var_int(ncid, gsize_varid, global_dimlen)))
+        return ret;
+
+    /* Write the size of the local array on each task. */
+    if ((PIOc_put_var_int(ncid, maplen_varid, task_maplen)))
+        return ret;
+
+    /* Write the map. */
+    if ((PIOc_put_var_int(ncid, map_varid, map)))
+        return ret;
+
+    /* Close the netCDF decomp file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/* Read the decomp information from a netCDF decomp file. This is an
+ * internal function.
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the name the decomp file will have.
+ * @param ndims pointer to int that will get number of dims in the
+ * data being described. Ignored if NULL.
+ * @param global_dimlen a pointer that gets an array, of size ndims,
+ * that will have the size of the global array in each
+ * dimension. Ignored if NULL, otherwise must be freed by caller.
+ * @param num_tasks pointer to int that gets the number of tasks the
+ * data are decomposed over. Ignored if NULL.
+ * @param task_maplen pointer that gets array of size num_tasks that
+ * gets the length of the map for each task. Ignored if NULL,
+ * otherwise must be freed by caller.
+ * @param max_maplen pointer to int that gets the maximum maplen for
+ * any task. Ignored if NULL.
+ * @param map pointer that gets a 2D array of size [num_tasks][max_maplen]
+ * that will have the 0-based mapping from local to global array
+ * elements. Ignored if NULL, otherwise must be freed by caller.
+ * @param title pointer that will get the contents of title attribute,
+ * if present. If present, title will be < PIO_MAX_NAME + 1 in
+ * length. Ignored if NULL.
+ * @param history pointer that will get the contents of history attribute,
+ * if present. If present, history will be < PIO_MAX_NAME + 1 in
+ * length. Ignored if NULL.
+ * @param source pointer that will get the contents of source
+ * attribute. Source will be < PIO_MAX_NAME + 1 in length. Ignored if
+ * NULL.
+ * @param version pointer that will get the contents of version
+ * attribute. It will be < PIO_MAX_NAME + 1 in length. Ignored if
+ * NULL.
+ * @param fortran_order int pointer that will get a 0 if this
+ * decomposition file uses C array ordering, 1 if it uses Fortran
+ * array ordering.
+ * @returns 0 for success, error code otherwise.
+ */
+int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen,
+                            int *num_tasks, int **task_maplen, int *max_maplen, int **map, char *title,
+                            char *history, char *source, char *version, int *fortran_order)
+{
+    iosystem_desc_t *ios;
+    int ncid;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "pioc_read_nc_decomp_int iosysid = %d filename = %s", iosysid, filename));
+
+    /* Open the netCDF decomp file. */
+    if ((ret = PIOc_open(iosysid, filename, NC_WRITE, &ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Read version attribute. */
+    char version_in[PIO_MAX_NAME + 1];
+    if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_VERSION_ATT_NAME, version_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "version_in = %s", version_in));
+    if (version)
+        strncpy(version, version_in, PIO_MAX_NAME + 1);
+
+    /* Read order attribute. */
+    char order_in[PIO_MAX_NAME + 1];
+    if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_ORDER_ATT_NAME, order_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "order_in = %s", order_in));
+    if (fortran_order)
+    {
+        if (!strncmp(order_in, DECOMP_C_ORDER_STR, PIO_MAX_NAME + 1))
+            *fortran_order = 0;
+        else if (!strncmp(order_in, DECOMP_FORTRAN_ORDER_STR, PIO_MAX_NAME + 1))
+            *fortran_order = 1;
+        else
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    }
+
+    /* Read attribute with the max map len. */
+    int max_maplen_in;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "max_maplen_in = %d", version_in));
+    if (max_maplen)
+        *max_maplen = max_maplen_in;
+
+    /* Read title attribute, if it is in the file. */
+    char title_in[NC_MAX_NAME + 1];
+    ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_TITLE_ATT_NAME, title_in);
+    if (ret == PIO_NOERR)
+    {
+        /* If the caller wants it, copy the title for them. */
+        if (title)
+            strncpy(title, title_in, PIO_MAX_NAME + 1);
+    }
+    else if (ret == PIO_ENOTATT)
+    {
+        /* No title attribute. */
+        if (title)
+            title[0] = '\0';
+    }
+    else
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Read history attribute, if it is in the file. */
+    char history_in[NC_MAX_NAME + 1];
+    ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_HISTORY_ATT_NAME, history_in);
+    if (ret == PIO_NOERR)
+    {
+        /* If the caller wants it, copy the history for them. */
+        if (history)
+            strncpy(history, history_in, PIO_MAX_NAME + 1);
+    }
+    else if (ret == PIO_ENOTATT)
+    {
+        /* No history attribute. */
+        if (history)
+            history[0] = '\0';
+    }
+    else
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Read source attribute. */
+    char source_in[NC_MAX_NAME + 1];
+    if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_SOURCE_ATT_NAME, source_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (source)
+        strncpy(source, source_in, PIO_MAX_NAME + 1);
+
+    /* Read dimension for the dimensions in the data. (Example: for 4D
+     * data we will need to store 4 dimension IDs.) */
+    int dim_dimid;
+    PIO_Offset ndims_in;
+    if ((ret = PIOc_inq_dimid(ncid, DECOMP_DIM_DIM, &dim_dimid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_inq_dim(ncid, dim_dimid, NULL, &ndims_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (ndims)
+        *ndims = ndims_in;
+
+    /* Read the global sizes of the array. */
+    int gsize_varid;
+    int global_dimlen_in[ndims_in];
+    if ((ret = PIOc_inq_varid(ncid, DECOMP_GLOBAL_SIZE_VAR_NAME, &gsize_varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_get_var_int(ncid, gsize_varid, global_dimlen_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (global_dimlen)
+    {
+        if (!(*global_dimlen = malloc(ndims_in * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int d = 0; d < ndims_in; d++)
+            (*global_dimlen)[d] = global_dimlen_in[d];
+    }
+
+    /* Read dimension for tasks. If we have 4 tasks, we need to store
+     * an array of length 4 with the size of the local array on each
+     * task. */
+    int task_dimid;
+    PIO_Offset num_tasks_in;
+    if ((ret = PIOc_inq_dimid(ncid, DECOMP_TASK_DIM_NAME, &task_dimid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_inq_dim(ncid, task_dimid, NULL, &num_tasks_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (num_tasks)
+        *num_tasks = num_tasks_in;
+
+    /* Read the length if the local array on each task. */
+    int maplen_varid;
+    int task_maplen_in[num_tasks_in];
+    if ((ret = PIOc_inq_varid(ncid, DECOMP_MAPLEN_VAR_NAME, &maplen_varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_get_var_int(ncid, maplen_varid, task_maplen_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (task_maplen)
+    {
+        if (!(*task_maplen = malloc(num_tasks_in * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int t = 0; t < num_tasks_in; t++)
+            (*task_maplen)[t] = task_maplen_in[t];
+    }
+
+    /* Read the map. */
+    int map_varid;
+    int map_in[num_tasks_in][max_maplen_in];
+    if ((ret = PIOc_inq_varid(ncid, DECOMP_MAP_VAR_NAME, &map_varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_get_var_int(ncid, map_varid, (int *)map_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (map)
+    {
+        if (!(*map = malloc(num_tasks_in * max_maplen_in * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int t = 0; t < num_tasks_in; t++)
+            for (int l = 0; l < max_maplen_in; l++)
+                (*map)[t * max_maplen_in + l] = map_in[t][l];
+    }
+
+    /* Close the netCDF decomp file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write the decomposition map to a file.
+ *
+ * @param file the filename to be used.
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the IO description.
+ * @param comm an MPI communicator.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+
+    LOG((1, "PIOc_write_decomp file = %s iosysid = %d ioid = %d", file, iosysid, ioid));
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    return PIOc_writemap(file, iodesc->ndims, iodesc->dimlen, iodesc->maplen, iodesc->map,
+                         comm);
+}
+
+/**
+ * Write the decomposition map to a file.
+ *
+ * @param file the filename
+ * @param ndims the number of dimensions
+ * @param gdims an array of dimension ids
+ * @param maplen the length of the map
+ * @param map the map array
+ * @param comm an MPI communicator.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
+                  PIO_Offset *map, MPI_Comm comm)
+{
+    int npes, myrank;
+    PIO_Offset *nmaplen = NULL;
+    MPI_Status status;
+    int i;
+    PIO_Offset *nmap;
+    int mpierr; /* Return code for MPI calls. */
+
+    LOG((1, "PIOc_writemap file = %s ndims = %d maplen = %d", file, ndims, maplen));
+
+    if ((mpierr = MPI_Comm_size(comm, &npes)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &myrank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "npes = %d myrank = %d", npes, myrank));
+
+    /* Allocate memory for the nmaplen. */
+    if (myrank == 0)
+        if (!(nmaplen = malloc(npes * sizeof(PIO_Offset))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Gather(&maplen, 1, PIO_OFFSET, nmaplen, 1, PIO_OFFSET, 0, comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    /* Only rank 0 writes the file. */
+    if (myrank == 0)
+    {
+        FILE *fp;
+
+        /* Open the file to write. */
+        if (!(fp = fopen(file, "w")))
+            return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__);
+
+        /* Write the version and dimension info. */
+        fprintf(fp,"version %d npes %d ndims %d \n", VERSNO, npes, ndims);
+        for (i = 0; i < ndims; i++)
+            fprintf(fp, "%d ", gdims[i]);
+        fprintf(fp, "\n");
+
+        /* Write the map. */
+        fprintf(fp, "0 %lld\n", nmaplen[0]);
+        for (i = 0; i < nmaplen[0]; i++)
+            fprintf(fp, "%lld ", map[i]);
+        fprintf(fp,"\n");
+
+        for (i = 1; i < npes; i++)
+        {
+            LOG((2, "creating nmap for i = %d", i));
+            nmap = (PIO_Offset *)malloc(nmaplen[i] * sizeof(PIO_Offset));
+
+            if ((mpierr = MPI_Send(&i, 1, MPI_INT, i, npes + i, comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if ((mpierr = MPI_Recv(nmap, nmaplen[i], PIO_OFFSET, i, i, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((2,"MPI_Recv map complete"));
+
+            fprintf(fp, "%d %lld\n", i, nmaplen[i]);
+            for (int j = 0; j < nmaplen[i]; j++)
+                fprintf(fp, "%lld ", nmap[j]);
+            fprintf(fp, "\n");
+
+            free(nmap);
+        }
+        /* Free memory for the nmaplen. */
+        free(nmaplen);
+        fprintf(fp, "\n");
+        print_trace(fp);
+
+        /* Close the file. */
+        fclose(fp);
+        LOG((2,"decomp file closed."));
+    }
+    else
+    {
+        LOG((2,"ready to MPI_Recv..."));
+        if ((mpierr = MPI_Recv(&i, 1, MPI_INT, 0, npes+myrank, comm, &status)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((2,"MPI_Recv got %d", i));
+        if ((mpierr = MPI_Send(map, maplen, PIO_OFFSET, 0, myrank, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((2,"MPI_Send map complete"));
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write the decomposition map to a file for F90.
+ *
+ * @param file the filename
+ * @param ndims the number of dimensions
+ * @param gdims an array of dimension ids
+ * @param maplen the length of the map
+ * @param map the map array
+ * @param comm an MPI communicator.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
+                           PIO_Offset maplen, const PIO_Offset *map, int f90_comm)
+{
+    return PIOc_writemap(file, ndims, gdims, maplen, (PIO_Offset *)map,
+                         MPI_Comm_f2c(f90_comm));
+}
+
+/**
+ * Create a new file using pio. This is an internal function that is
+ * called by both PIOc_create() and PIOc_createfile(). Input
+ * parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid A defined pio system ID, obtained from
+ * PIOc_InitIntercomm() or PIOc_InitAsync().
+ * @param ncidp A pointer that gets the ncid of the newly created
+ * file.
+ * @param iotype A pointer to a pio output format. Must be one of
+ * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or
+ * PIO_IOTYPE_NETCDF4P.
+ * @param filename The filename to create.
+ * @param mode The netcdf mode for the create operation.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_createfile
+ */
+int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
+                        int mode)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;              /* Return code from function calls. */
+
+    /* Get the IO system info from the iosysid. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* User must provide valid input for these parameters. */
+    if (!ncidp || !iotype || !filename || strlen(filename) > NC_MAX_NAME)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* A valid iotype must be specified. */
+    if (!iotype_is_valid(*iotype))
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_createfile iosysid = %d iotype = %d filename = %s mode = %d",
+         iosysid, *iotype, filename, mode));
+
+    /* Allocate space for the file info. */
+    if (!(file = calloc(sizeof(file_desc_t), 1)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Fill in some file values. */
+    file->fh = -1;
+    file->iosystem = ios;
+    file->iotype = *iotype;
+    file->buffer.ioid = -1;
+    for (int i = 0; i < PIO_MAX_VARS; i++)
+    {
+        file->varlist[i].record = -1;
+        file->varlist[i].ndims = -1;
+	file->varlist[i].iobuf = NULL;
+	file->varlist[i].fillbuf = NULL;
+    }
+    file->mode = mode;
+
+    /* Set to true if this task should participate in IO (only true for
+     * one task with netcdf serial files. */
+    if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF ||
+        ios->io_rank == 0)
+        file->do_io = 1;
+
+    LOG((2, "file->do_io = %d ios->async_interface = %d", file->do_io, ios->async_interface));
+
+    /* If async is in use, and this is not an IO task, bcast the
+     * parameters. */
+    if (ios->async_interface)
+    {
+        int msg = PIO_MSG_CREATE_FILE;
+        size_t len = strlen(filename);
+
+        if (!ios->ioproc)
+        {
+            /* Send the message to the message handler. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the parameters of the function call. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "len = %d filename = %s iotype = %d mode = %d", len, filename,
+                 file->iotype, file->mode));
+        }
+
+        /* Handle MPI errors. */
+        LOG((2, "handling mpi errors mpierr = %d", mpierr));
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this task is in the IO component, do the IO. */
+    if (ios->ioproc)
+    {
+        switch (file->iotype)
+        {
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            file->mode = file->mode |  NC_MPIIO | NC_NETCDF4;
+            LOG((2, "Calling nc_create_par io_comm = %d mode = %d fh = %d",
+                 ios->io_comm, file->mode, file->fh));
+            ierr = nc_create_par(filename, file->mode, ios->io_comm, ios->info, &file->fh);
+            LOG((2, "nc_create_par returned %d file->fh = %d", ierr, file->fh));
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+            file->mode = file->mode | NC_NETCDF4;
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (!ios->io_rank)
+            {
+                LOG((2, "Calling nc_create mode = %d", file->mode));
+                ierr = nc_create(filename, file->mode, &file->fh);
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            LOG((2, "Calling ncmpi_create mode = %d", file->mode));
+            ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &file->fh);
+            if (!ierr)
+                ierr = ncmpi_buffer_attach(file->fh, pio_buffer_size_limit);
+            break;
+#endif
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    /* If there was an error, free the memory we allocated and handle error. */
+    if (ierr)
+    {
+        free(file);
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+    }
+
+    /* Broadcast mode to all tasks. */
+    if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    /* This flag is implied by netcdf create functions but we need
+       to know if its set. */
+    file->mode = file->mode | PIO_WRITE;
+
+    /* Assign the PIO ncid, necessary because files may be opened
+     * on mutilple iosystems, causing the underlying library to
+     * reuse ncids. Hilarious confusion ensues. */
+    file->pio_ncid = pio_next_ncid++;
+    LOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid));
+
+    /* Return the ncid to the caller. */
+    *ncidp = file->pio_ncid;
+
+    /* Add the struct with this files info to the global list of
+     * open files. */
+    pio_add_to_file_list(file);
+
+    LOG((2, "Created file %s file->fh = %d file->pio_ncid = %d", filename,
+         file->fh, file->pio_ncid));
+
+    return ierr;
+}
+
+/**
+ * Open an existing file using PIO library. This is an internal
+ * function. Depending on the value of the retry parameter, a failed
+ * open operation will be handled differently. If retry is non-zero,
+ * then a failed attempt to open a file with netCDF-4 (serial or
+ * parallel), or parallel-netcdf will be followed by an attempt to
+ * open the file as a serial classic netCDF file. This is an important
+ * feature to some NCAR users. The functionality is exposed to the
+ * user as PIOc_openfile() (which does the retry), and PIOc_open()
+ * (which does not do the retry).
+ *
+ * Input parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid: A defined pio system descriptor (input)
+ * @param ncidp: A pio file descriptor (output)
+ * @param iotype: A pio output format (input)
+ * @param filename: The filename to open
+ * @param mode: The netcdf mode for the open operation
+ * @param retry: non-zero to automatically retry with netCDF serial
+ * classic.
+ *
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_openfile
+ */
+int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
+                        int mode, int retry)
+{
+    iosystem_desc_t *ios;  /** Pointer to io system information. */
+    file_desc_t *file;     /** Pointer to file information. */
+    int imode;  /** internal mode val for netcdf4 file open */
+    int mpierr = MPI_SUCCESS, mpierr2;  /** Return code from MPI function codes. */
+    int ierr = PIO_NOERR;  /** Return code from function calls. */
+
+    /* Get the IO system info from the iosysid. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* User must provide valid input for these parameters. */
+    if (!ncidp || !iotype || !filename)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((2, "PIOc_openfile_retry iosysid = %d iotype = %d filename = %s mode = %d retry = %d",
+         iosysid, *iotype, filename, mode, retry));
+
+    /* Allocate space for the file info. */
+    if (!(file = calloc(sizeof(*file), 1)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Fill in some file values. */
+    file->fh = -1;
+    file->iotype = *iotype;
+    file->iosystem = ios;
+    file->mode = mode;
+
+    for (int i = 0; i < PIO_MAX_VARS; i++)
+    {
+        file->varlist[i].record = -1;
+        file->varlist[i].ndims = -1;
+    }
+
+    /* Set to true if this task should participate in IO (only true
+     * for one task with netcdf serial files. */
+    if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF ||
+        ios->io_rank == 0)
+        file->do_io = 1;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        int msg = PIO_MSG_OPEN_FILE;
+        size_t len = strlen(filename);
+
+        if (!ios->ioproc)
+        {
+            /* Send the message to the message handler. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the parameters of the function call. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        switch (file->iotype)
+        {
+#ifdef _NETCDF4
+
+        case PIO_IOTYPE_NETCDF4P:
+#ifdef _MPISERIAL
+            ierr = nc_open(filename, file->mode, &file->fh);
+#else
+            imode = file->mode |  NC_MPIIO;
+            ierr = nc_open_par(filename, imode, ios->io_comm, ios->info, &file->fh);
+            if (ierr == PIO_NOERR)
+                file->mode = imode;
+            LOG((2, "PIOc_openfile_retry:nc_open_par filename = %s mode = %d imode = %d ierr = %d",
+                 filename, file->mode, imode, ierr));
+#endif
+            break;
+
+        case PIO_IOTYPE_NETCDF4C:
+            if (ios->io_rank == 0)
+            {
+                imode = file->mode | NC_NETCDF4;
+                ierr = nc_open(filename, imode, &file->fh);
+                if (ierr == PIO_NOERR)
+                    file->mode = imode;
+            }
+            break;
+
+#endif
+
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank == 0)
+                ierr = nc_open(filename, file->mode, &file->fh);
+            break;
+
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &file->fh);
+
+            // This should only be done with a file opened to append
+            if (ierr == PIO_NOERR && (file->mode & PIO_WRITE))
+            {
+                if (ios->iomaster == MPI_ROOT)
+                    LOG((2, "%d Setting IO buffer %ld", __LINE__, pio_buffer_size_limit));
+                ierr = ncmpi_buffer_attach(file->fh, pio_buffer_size_limit);
+            }
+            LOG((2, "ncmpi_open(%s) : fd = %d", filename, file->fh));
+            break;
+#endif
+
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+
+        /* If the caller requested a retry, and we failed to open a
+           file due to an incompatible type of NetCDF, try it once
+           with just plain old basic NetCDF. */
+        if (retry)
+        {
+            LOG((2, "retry error code ierr = %d io_rank %d", ierr, ios->io_rank));
+            if ((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF))
+            {
+                if (ios->iomaster == MPI_ROOT)
+                    printf("PIO2 pio_file.c retry NETCDF\n");
+
+                /* reset ierr on all tasks */
+                ierr = PIO_NOERR;
+
+                /* reset file markers for NETCDF on all tasks */
+                file->iotype = PIO_IOTYPE_NETCDF;
+
+                /* open netcdf file serially on main task */
+                if (ios->io_rank == 0)
+                    ierr = nc_open(filename, file->mode, &file->fh);
+                else
+                    file->do_io = 0;
+            }
+            LOG((2, "retry nc_open(%s) : fd = %d, iotype = %d, do_io = %d, ierr = %d", filename, file->fh, file->iotype, file->do_io, ierr));
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    LOG((2, "Bcasting error code ierr = %d ios->ioroot = %d ios->my_comm = %d", ierr, ios->ioroot,
+         ios->my_comm));
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "Bcast error code ierr = %d", ierr));
+
+    /* If there was an error, free allocated memory and deal with the error. */
+    if (ierr)
+    {
+        free(file);
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+    }
+    LOG((2, "error code Bcast complete ierr = %d ios->my_comm = %d", ierr, ios->my_comm));
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    /* Create the ncid that the user will see. This is necessary
+     * because otherwise ncids will be reused if files are opened
+     * on multiple iosystems. */
+    file->pio_ncid = pio_next_ncid++;
+
+    /* Return the PIO ncid to the user. */
+    *ncidp = file->pio_ncid;
+
+    /* Add this file to the list of currently open files. */
+    pio_add_to_file_list(file);
+
+    LOG((2, "Opened file %s file->pio_ncid = %d file->fh = %d ierr = %d",
+         filename, file->pio_ncid, file->fh, ierr));
+
+    return ierr;
+}
+
+/**
+ * Internal function to provide inq_type function for pnetcdf.
+ *
+ * @param ncid ignored because pnetcdf does not have user-defined
+ * types.
+ * @param xtype type to learn about.
+ * @param name pointer that gets name of type. Ignored if NULL.
+ * @param sizep pointer that gets size of type. Ignored if NULL.
+ * @returns 0 on success, error code otherwise.
+ */
+int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
+                          PIO_Offset *sizep)
+{
+    int typelen;
+
+    switch (xtype)
+    {
+    case NC_UBYTE:
+    case NC_BYTE:
+    case NC_CHAR:
+        typelen = 1;
+        break;
+    case NC_SHORT:
+    case NC_USHORT:
+        typelen = 2;
+        break;
+    case NC_UINT:
+    case NC_INT:
+    case NC_FLOAT:
+        typelen = 4;
+        break;
+    case NC_UINT64:
+    case NC_INT64:
+    case NC_DOUBLE:
+        typelen = 8;
+        break;
+    default:
+        return PIO_EBADTYPE;
+    }
+
+    /* If pointers were supplied, copy results. */
+    if (sizep)
+        *sizep = typelen;
+    if (name)
+        strcpy(name, "some type");
+
+    return PIO_NOERR;
+}
+
+/**
+ * This is an internal function that handles both PIOc_enddef and
+ * PIOc_redef.
+ *
+ * @param ncid the ncid of the file to enddef or redef
+ * @param is_enddef set to non-zero for enddef, 0 for redef.
+ * @returns PIO_NOERR on success, error code on failure. */
+int pioc_change_def(int ncid, int is_enddef)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    LOG((2, "pioc_change_def ncid = %d is_enddef = %d", ncid, is_enddef));
+
+    /* Find the info about this file. When I check the return code
+     * here, some tests fail. ???*/
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF;
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((3, "pioc_change_def ncid = %d mpierr = %d", ncid, mpierr));
+        }
+
+        /* Handle MPI errors. */
+        LOG((3, "pioc_change_def handling MPI errors"));
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    LOG((3, "pioc_change_def ios->ioproc = %d", ios->ioproc));
+    if (ios->ioproc)
+    {
+        LOG((3, "pioc_change_def calling netcdf function file->fh = %d file->do_io = %d iotype = %d",
+             file->fh, file->do_io, file->iotype));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            if (is_enddef)
+                ierr = ncmpi_enddef(file->fh);
+            else
+                ierr = ncmpi_redef(file->fh);
+        }
+#endif /* _PNETCDF */
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            if (is_enddef)
+            {
+                LOG((3, "pioc_change_def calling nc_enddef file->fh = %d", file->fh));
+                ierr = nc_enddef(file->fh);
+            }
+            else
+                ierr = nc_redef(file->fh);
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    LOG((3, "pioc_change_def bcasting return code ierr = %d", ierr));
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((3, "pioc_change_def succeeded"));
+
+    return ierr;
+}
+
+/**
+ * Check whether an IO type is valid for the build.
+ *
+ * @param iotype the IO type to check
+ * @returns 0 if valid, non-zero otherwise.
+ */
+int iotype_is_valid(int iotype)
+{
+    /* Assume it's not valid. */
+    int ret = 0;
+
+    /* All builds include netCDF. */
+    if (iotype == PIO_IOTYPE_NETCDF)
+        ret++;
+
+    /* Some builds include netCDF-4. */
+#ifdef _NETCDF4
+    if (iotype == PIO_IOTYPE_NETCDF4C || iotype == PIO_IOTYPE_NETCDF4P)
+        ret++;
+#endif /* _NETCDF4 */
+
+    /* Some builds include pnetcdf. */
+    if (iotype == PIO_IOTYPE_PNETCDF)
+        ret++;
+#ifdef _PNETCDF
+#endif /* _PNETCDF */
+
+    return ret;
+}
+
+/**
+ * Internal function to compare rearranger flow control options.
+ *
+ * @param opt pointer to rearranger flow control options to compare.
+ * @param exp_opt pointer to rearranger flow control options with
+ * expected values.
+ * @return true if values in opt == values in exp_opt, false
+ * otherwise.
+ */
+bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt,
+                            const rearr_comm_fc_opt_t *exp_opt)
+{
+    bool is_same = true;
+
+    assert(opt && exp_opt);
+
+    if (opt->enable_hs != exp_opt->enable_hs)
+    {
+        LOG((1, "Warning rearranger enable_hs = %s, expected = %s",
+             opt->enable_hs ? "TRUE" : "FALSE", exp_opt->enable_hs ? "TRUE" : "FALSE"));
+        is_same = false;
+    }
+
+    if (opt->enable_isend != exp_opt->enable_isend)
+    {
+        LOG((1, "Warning rearranger enable_isend = %s, expected = %s",
+             opt->enable_isend ? "TRUE" : "FALSE", exp_opt->enable_isend ? "TRUE" : "FALSE"));
+        is_same = false;
+    }
+
+    if (opt->max_pend_req != exp_opt->max_pend_req)
+    {
+        LOG((1, "Warning rearranger max_pend_req = %d, expected = %d",
+             opt->max_pend_req, exp_opt->max_pend_req));
+        is_same = false;
+    }
+
+    return is_same;
+}
+
+/**
+ * Internal function to compare rearranger options.
+ *
+ * @param rearr_opts pointer to rearranger options to compare
+ * @param exp_rearr_opts pointer to rearranger options with the
+ * expected value
+ * @return true if values in rearr_opts == values in exp_rearr_opts
+ * false otherwise
+ */
+bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts)
+{
+    bool is_same = true;
+
+    assert(rearr_opts && exp_rearr_opts);
+
+    if (rearr_opts->comm_type != exp_rearr_opts->comm_type)
+    {
+        LOG((1, "Warning rearranger comm_type = %d, expected = %d. ", rearr_opts->comm_type,
+             exp_rearr_opts->comm_type));
+        is_same = false;
+    }
+
+    if (rearr_opts->fcd != exp_rearr_opts->fcd)
+    {
+        LOG((1, "Warning rearranger fcd = %d, expected = %d. ", rearr_opts->fcd,
+             exp_rearr_opts->fcd));
+        is_same = false;
+    }
+
+    is_same = is_same && cmp_rearr_comm_fc_opts(&(rearr_opts->comm_fc_opts_comp2io),
+                                                &(exp_rearr_opts->comm_fc_opts_comp2io));
+    is_same = is_same && cmp_rearr_comm_fc_opts(&(rearr_opts->comm_fc_opts_io2comp),
+                                                &(exp_rearr_opts->comm_fc_opts_io2comp));
+
+    return is_same;
+}
+
+/**
+ * Internal function to reset rearranger opts in iosystem to valid values.
+ * The old default for max pending requests was DEF_P2P_MAXREQ = 64.
+ *
+ * @param ios pointer to iosystem descriptor
+ */
+void check_and_reset_rearr_opts(iosystem_desc_t *ios)
+{
+    /* Disable handshake/isend and set max_pend_req to unlimited */
+    const rearr_comm_fc_opt_t def_comm_nofc_opts =
+        { false, false, PIO_REARR_COMM_UNLIMITED_PEND_REQ };
+    /* Disable handshake /isend and set max_pend_req = 0 to turn off throttling */
+    const rearr_comm_fc_opt_t def_coll_comm_fc_opts = { false, false, 0 };
+    const rearr_opt_t def_coll_rearr_opts = {
+        PIO_REARR_COMM_COLL,
+        PIO_REARR_COMM_FC_2D_DISABLE,
+        def_coll_comm_fc_opts,
+        def_coll_comm_fc_opts
+    };
+
+    assert(ios);
+
+    /* Reset to defaults, if needed (user did not set it correctly) */
+    if (ios->rearr_opts.comm_type == PIO_REARR_COMM_COLL)
+    {
+        /* Compare and log the user and default rearr opts for coll. */
+        cmp_rearr_opts(&(ios->rearr_opts), &def_coll_rearr_opts);
+        /* Hard reset flow control options. */
+        ios->rearr_opts = def_coll_rearr_opts;
+    }
+    else if (ios->rearr_opts.comm_type == PIO_REARR_COMM_P2P)
+    {
+        if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_2D_DISABLE)
+        {
+            /* Compare and log user and default opts. */
+            cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_comp2io),
+                                   &def_comm_nofc_opts);
+            cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_io2comp),
+                                   &def_comm_nofc_opts);
+            /* Hard reset flow control opts to defaults. */
+            ios->rearr_opts.comm_fc_opts_comp2io = def_comm_nofc_opts;
+            ios->rearr_opts.comm_fc_opts_io2comp = def_comm_nofc_opts;
+        }
+        else if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_1D_COMP2IO)
+        {
+            /* Compare and log user and default opts. */
+            cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_io2comp),
+                                   &def_comm_nofc_opts);
+            /* Hard reset io2comp dir to defaults. */
+            ios->rearr_opts.comm_fc_opts_io2comp = def_comm_nofc_opts;
+        }
+        else if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_1D_IO2COMP)
+        {
+            /* Compare and log user and default opts. */
+            cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_comp2io),
+                                   &def_comm_nofc_opts);
+            /* Hard reset comp2io dir to defaults. */
+            ios->rearr_opts.comm_fc_opts_comp2io = def_comm_nofc_opts;
+        }
+        /* Don't reset if flow control is enabled in both directions
+         * by user. */
+    }
+}
+
+/**
+ * Set the rearranger options associated with an iosystem
+ *
+ * @param comm_type Type of communication (pt2pt/coll) used
+ * by the rearranger. See PIO_REARR_COMM_TYPE for more detail.
+ * Possible values are :
+ * PIO_REARR_COMM_P2P (Point to point communication)
+ * PIO_REARR_COMM_COLL (Collective communication)
+ * @param fcd Flow control direction for the rearranger.
+ * See PIO_REARR_COMM_FC_DIR for more detail.
+ * Possible values are :
+ * PIO_REARR_COMM_FC_2D_ENABLE : Enable flow control from
+ * compute processes to io processes and vice versa
+ * PIO_REARR_COMM_FC_1D_COMP2IO : Enable flow control from
+ * compute processes to io processes (only)
+ * PIO_REARR_COMM_FC_1D_IO2COMP : Enable flow control from
+ * io processes to compute processes (only)
+ * PIO_REARR_COMM_FC_2D_DISABLE : Disable flow control from
+ * compute processes to io processes and vice versa.
+ * @param enable_hs_c2i Enable handshake while rearranging
+ * data, from compute to io processes
+ * @param enable_isend_c2i Enable isends while rearranging
+ * data, from compute to io processes
+ * @param max_pend_req_c2i Maximum pending requests during
+ * data rearragment from compute processes to io processes
+ * @param enable_hs_i2c Enable handshake while rearranging
+ * data, from io to compute processes
+ * @param enable_isend_i2c Enable isends while rearranging
+ * data, from io to compute processes
+ * @param max_pend_req_i2c Maximum pending requests during
+ * data rearragment from io processes to compute processes
+ * @param iosysidp index of the defined system descriptor
+ * @return 0 on success, otherwise a PIO error code.
+ */
+int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i,
+                        bool enable_isend_c2i, int max_pend_req_c2i,
+                        bool enable_hs_i2c, bool enable_isend_i2c,
+                        int max_pend_req_i2c)
+{
+    iosystem_desc_t *ios;
+    int ret = PIO_NOERR;
+    rearr_opt_t user_rearr_opts = {
+        comm_type, fcd,
+        {enable_hs_c2i,enable_isend_c2i, max_pend_req_c2i},
+        {enable_hs_i2c, enable_isend_i2c, max_pend_req_i2c}
+    };
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    ios->rearr_opts = user_rearr_opts;
+
+    /* Perform sanity checks on the user supplied values */
+    check_and_reset_rearr_opts(ios);
+
+    return ret;
+}
diff --git a/src/externals/pio2/src/clib/topology.c b/src/externals/pio2/src/clib/topology.c
new file mode 100644
index 00000000000..f8e1fc016be
--- /dev/null
+++ b/src/externals/pio2/src/clib/topology.c
@@ -0,0 +1,363 @@
+#if defined(BGQDONT)
+
+#include 
+#include 
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#define   Personality                    Personality_t
+
+int rank;
+int np;
+int my_name_len;
+char my_name[255];
+
+void identity(const MPI_Comm comm, int *iotask)
+{
+    MPIX_Hardware_t hw;
+    char message[100];
+    /* Number of MPI tasks per Pset */
+    int coreId;
+    int *TasksPerPset;
+    int *tmp;
+    int i,ierr;
+    Personality_t pers;
+
+    MPI_Comm_rank(comm,&rank);
+    MPI_Comm_size(comm,&np);
+    MPI_Get_processor_name(my_name, &my_name_len);
+
+
+    MPIX_Hardware(&hw);
+
+    Kernel_GetPersonality(&pers, sizeof(pers));
+
+    int numIONodes,numPsets,numNodesInPset,rankInPset;
+    int numpsets, psetID, psetsize, psetrank;
+
+    bgq_pset_info (comm, &numpsets, &psetID, &psetsize, &psetrank);
+
+    numIONodes = numpsets;
+    numNodesInPset = psetsize;
+    rankInPset = rank;
+    numPsets = numpsets;
+
+    if(rank == 0) { printf("number of IO nodes in block: %i \n",numIONodes);}
+    if(rank == 0) { printf("number of Psets in block : %i \n",numPsets);}
+    if(rank == 0) { printf("number of compute nodes in Pset: %i \n",numNodesInPset);}
+
+    int psetNum;
+    psetNum = psetID;
+
+#ifdef DEBUG
+    if((*iotask)>0) {
+        printf( "%04i (%-50s %s) %i yes\n", rank, my_name, message, psetNum );
+    } else {
+        printf( "%04i (%-50s %s) %i --\n", rank, my_name, message, psetNum);
+    }
+    printf("MPI task %6i is rank %3i in Pset: %3i \n",rank, rankInPset,psetNum);
+#endif
+    /* Determine which core on node....  I don't want to put more than one io-task per node */
+    coreId = Kernel_PhysicalProcessorID ();
+
+    TasksPerPset = malloc(numPsets*sizeof(int));
+    tmp = malloc(numPsets*sizeof(int));
+    for(i=0;i 0) {
+
+
+        Kernel_GetPersonality(&pers, sizeof(pers));
+
+        int numIONodes,numPsets,numNodesInPset,rankInPset;
+        int numiotasks_per_node,remainder,numIONodes_per_pset;
+        int lstride;
+
+        /* Number of computational nodes in processor set */
+
+        int numpsets, psetID, psetsize, psetrank;
+        bgq_pset_info (comm,&numpsets, &psetID, &psetsize, &psetrank);
+        numIONodes = numpsets;
+        numNodesInPset = psetsize;
+
+        /*     printf("Determine io tasks: me %i : nodes in pset= %i ionodes = %i\n", rank, numNodesInPset, numIONodes); */
+
+        if((*numiotasks) < 0 ) {
+            /* negative numiotasks value indicates that this is the number per IO-node */
+            (*numiotasks) = - (*numiotasks);
+            if((*numiotasks) > numNodesInPset) {
+                numiotasks_per_node = numNodesInPset;
+            } else  {
+                numiotasks_per_node = (*numiotasks);
+            }
+            remainder = 0;
+        } else if ((*numiotasks) > 0 ) {
+            /* balance the number of iotasks to number of IO nodes */
+            numiotasks_per_node = floor((float)(*numiotasks)/ (float) numIONodes);
+            /* put a minumum here so that we have a chance  - though this may be too low */
+            if (numiotasks_per_node < 1) {
+                numiotasks_per_node = 1;
+                *numiotasks = numIONodes;
+            }
+            remainder = (*numiotasks) - numiotasks_per_node * numIONodes;
+        } else if ((*numiotasks) == 0 ) {
+            if((*stride) > 0) {
+                numiotasks_per_node = numNodesInPset/(*stride);
+                if (numiotasks_per_node < 1) {
+                    numiotasks_per_node = 1;
+                    *numiotasks = numIONodes;
+                }
+            } else {
+                numiotasks_per_node = 8;  /* default number of IO-client per IO-node is not otherwise specificied */
+            }
+            remainder = 0;
+        }
+
+        /* number of IO nodes with a larger number of io-client per io-node */
+        if(remainder > 0) {
+            if(rank ==0) {printf("Unbalanced IO-configuration: %i IO-nodes have %i IO-clients : %i IO-nodes have %i IO-clients \n",
+                                 remainder, numiotasks_per_node+1, numIONodes-remainder,numiotasks_per_node);}
+            lstride = min(np,floor((float)numNodesInPset/(float)(numiotasks_per_node+1)));
+        } else {
+            if(rank == 0) {
+                printf("Balanced IO-configuration: %i IO-nodes have %i IO-clients\n",numIONodes-remainder, numiotasks_per_node);
+            }
+            lstride = min(np,floor((float)numNodesInPset/(float)numiotasks_per_node));
+        }
+
+        /* Number of processor sets */
+        numPsets = numpsets;
+
+        /* number of IO nodes in processor set (I need to add
+           code to deal with the case where numIONodes_per_pset != 1 works
+           correctly) */
+        numIONodes_per_pset = numIONodes/numPsets;
+
+        /* Determine which core on node....  I don't want to put more than one io-task per node */
+        coreId = Kernel_PhysicalProcessorID ();
+
+        /* What is the rank of this node in the processor set */
+
+        psetNum = psetID;
+        rankInPset = psetrank;
+
+        /* printf("Pset #: %i has %i nodes in Pset; base = %i\n",psetNum,numNodesInPset, *base); */
+
+        (*iamIOtask) = false;   /* initialize to zero */
+
+        if (numiotasks_per_node == numNodesInPset)(*base) = 0;  /* Reset the base to 0 if we are using all tasks */
+
+
+        /* start stridding MPI tasks from base task */
+        iam = max(0,rankInPset-(*base));
+        if (iam >= 0)  {
+            /* mark tasks that will be IO-tasks  or IO-clients */
+            /*    printf("iam = %d lstride = %d coreID = %d\n",iam,lstride,coreId); */
+            if((iam % lstride == 0) && (coreId == 0) ) {  /* only io tasks indicated by stride and coreId = 0 */
+                if((iam/lstride) < numiotasks_per_node) {
+                    /* only set the first (numiotasks_per_node - 1) tasks */
+                    (*iamIOtask) = true;
+                } else if ((iam/lstride) == numiotasks_per_node) {
+                    /*  If there is an uneven number of io-clients to io-nodes
+                        allocate the first remainder - 1 processor sets to
+                        have a total of numiotasks_per_node */
+                    if(psetNum < remainder) {(*iamIOtask) = true;
+                    };
+                }
+            }
+            /*       printf("comm = %d iam = %d lstride = %d coreID = %d iamIOtask = %i \n",comm, iam,lstride,coreId,(*iamIOtask)); */
+        }
+    }else{
+        /* We are not doing rearrangement.... so all tasks are io-tasks */
+        (*iamIOtask) = true;
+    }
+
+    /*printf("comm = %d myrank = %i iotask = %i \n", comm, rank, (*iamIOtask));*/
+
+    /* now we need to correctly determine the numiotasks */
+    MPI_Allreduce(iamIOtask, &task_count, 1, MPI_INT, MPI_SUM, comm);
+
+    (*numiotasks) = task_count;
+
+
+}
+
+int bgq_ion_id (void)
+{
+    int iA,  iB,  iC,  iD,  iE;                /* The local node's coordinates  */
+    int nA,  nB,  nC,  nD,  nE;                /* Size of each torus dimension  */
+    int brA, brB, brC, brD, brE;               /* The bridge node's coordinates */
+    int io_node_route_id;
+
+    Personality_t personality;
+
+    Kernel_GetPersonality(&personality, sizeof(personality));
+
+    iA  = personality.Network_Config.Acoord;
+    iB  = personality.Network_Config.Bcoord;
+    iC  = personality.Network_Config.Ccoord;
+    iD  = personality.Network_Config.Dcoord;
+    iE  = personality.Network_Config.Ecoord;
+
+    nA  = personality.Network_Config.Anodes;
+    nB  = personality.Network_Config.Bnodes;
+    nC  = personality.Network_Config.Cnodes;
+    nD  = personality.Network_Config.Dnodes;
+    nE  = personality.Network_Config.Enodes;
+
+    brA = personality.Network_Config.cnBridge_A;
+    brB = personality.Network_Config.cnBridge_B;
+    brC = personality.Network_Config.cnBridge_C;
+    brD = personality.Network_Config.cnBridge_D;
+    brE = personality.Network_Config.cnBridge_E;
+
+/*
+ * This is the bridge node, numbered in ABCDE order, E increments first.
+ * It is considered the unique "io node route identifer" because each
+ * bridge node only has one torus link to one io node.
+ */
+
+    io_node_route_id = brE + brD*nE + brC*nD*nE + brB*nC*nD*nE + brA*nB*nC*nD*nE;
+
+    return io_node_route_id;
+
+}
+
+
+
+int bgq_pset_info (MPI_Comm comm, int* tot_pset, int* psetID, int* pset_size, int* rank_in_pset)
+{
+    MPI_Comm comp_comm, pset_comm, bridge_comm;
+    int comp_rank, status, key, bridge_root, tot_bridges, cur_pset, itr, t_buf;
+    int temp_id, rem_psets;
+    MPI_Status mpi_status;
+
+    MPI_Comm_rank (comm, &comp_rank);
+
+    status = MPI_Comm_dup ( comm, &comp_comm);
+    if ( MPI_SUCCESS != status)
+    {
+        printf(" Error duplicating communicator \n");
+        MPI_Abort(comm, status);
+    }
+
+    // Compute the ION BridgeNode ID
+    key = bgq_ion_id ();
+
+    // Create the pset_comm per bridge node
+    status = MPI_Comm_split ( comp_comm, key, comp_rank, &pset_comm);
+    if ( MPI_SUCCESS != status)
+    {
+        printf(" Error splitting communicator \n");
+        MPI_Abort(comm, status);
+    }
+
+    // Calculate the rank in pset and pset size
+    MPI_Comm_rank (pset_comm, rank_in_pset);
+    MPI_Comm_size (pset_comm, pset_size);
+
+    // Create the Bridge root nodes communicator
+    bridge_root = 0;
+    if (0 == *rank_in_pset)
+        bridge_root = 1;
+
+    // Calculate the total number of bridge nodes / psets
+    tot_bridges = 0;
+    MPI_Allreduce (&bridge_root, &tot_bridges, 1, MPI_INT, MPI_SUM, comm);
+
+    *tot_pset = tot_bridges;
+
+    // Calculate the Pset ID
+    cur_pset = 0;
+    rem_psets = tot_bridges;
+    if ((0 == comp_rank) && (bridge_root ==1))
+    {
+        *psetID = 0;
+        rem_psets = tot_bridges-1;
+        cur_pset++;
+    }
+
+    t_buf = 0; // Dummy value
+    if (0 == comp_rank)
+    {
+        for (itr = 0; itr < rem_psets; itr++)
+        {
+            MPI_Recv (&t_buf,1, MPI_INT,  MPI_ANY_SOURCE, MPI_ANY_TAG,comm, &mpi_status);
+            MPI_Send (&cur_pset, 1, MPI_INT, mpi_status.MPI_SOURCE, 0, comm);
+            cur_pset++;
+        }
+    }
+
+    if ((1 == bridge_root) && ( 0 != comp_rank))
+    {
+        MPI_Send (&t_buf, 1, MPI_INT, 0, 0, comm);
+        MPI_Recv (&temp_id,1, MPI_INT, 0, 0, comm, &mpi_status);
+
+        *psetID = temp_id;
+        /*printf (" Pset ID is %d \n", *psetID);*/
+    }
+    // Broadcast the PSET ID to all ranks in the psetcomm
+    MPI_Bcast ( psetID, 1, MPI_INT, 0, pset_comm);
+
+    // Free the split comm
+    MPI_Comm_free (&pset_comm);
+
+    MPI_Barrier (comm);
+
+    return 0;
+}
+
+#endif
diff --git a/src/externals/pio2/src/flib/CMakeLists.txt b/src/externals/pio2/src/flib/CMakeLists.txt
new file mode 100644
index 00000000000..3d2c1711252
--- /dev/null
+++ b/src/externals/pio2/src/flib/CMakeLists.txt
@@ -0,0 +1,268 @@
+cmake_minimum_required (VERSION 2.8.12)
+project (PIOF Fortran)
+include (CheckFunctionExists)
+include (ExternalProject)
+include (LibCheck)
+
+#==============================================================================
+#  DEFINE THE TARGET
+#==============================================================================
+
+set (PIO_Fortran_SRCS pio_nf.F90
+  pio.F90
+  pio_kinds.F90
+  pio_types.F90
+  piolib_mod.F90
+  pio_support.F90)
+
+set (PIO_GenF90_SRCS pionfatt_mod.F90
+  pionfput_mod.F90
+  pionfget_mod.F90
+  piodarray.F90)
+
+set (PIO_Fortran_MODS ${CMAKE_CURRENT_BINARY_DIR}/pio.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_nf.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_types.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/piolib_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pionfget_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_kinds.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_support.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/piodarray.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pionfatt_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pionfput_mod.mod)
+
+add_library (piof ${PIO_Fortran_SRCS} ${PIO_GenF90_SRCS})
+if (NOT PIO_ENABLE_FORTRAN)
+  set_target_properties(piof PROPERTIES EXCLUDE_FROM_ALL TRUE)
+endif ()
+
+# Include flib source and binary directories (for Fortran modules)
+target_include_directories (piof
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
+  PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+# System and compiler CPP directives
+target_compile_definitions (piof
+  PUBLIC ${CMAKE_SYSTEM_DIRECTIVE})
+target_compile_definitions (piof
+  PUBLIC ${CMAKE_Fortran_COMPILER_DIRECTIVE})
+
+# Compiler-specific compile options
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (piof
+    PRIVATE -ffree-line-length-none)
+elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (piof
+  #        PRIVATE -mismatch_all)
+endif()
+
+# Look for c_sizeof capability
+check_macro (Fortran_CSIZEOF
+  NAME TryCSizeOf.f90
+  HINTS ${CMAKE_MODULE_PATH}
+  COMMENT "whether Fortran compiler supports c_sizeof")
+if (NOT Fortran_CSIZEOF)
+  target_compile_definitions (piof
+    PUBLIC NO_C_SIZEOF)
+endif()
+
+# Look for filesystem hints
+if (PIO_FILESYSTEM_HINTS)
+  if (PIO_FILESYSTEM_HINTS STREQUAL lustre)
+    message (STATUS "PIO will use lustre filesystem hints")
+    target_compile_definitions (piof
+      PUBLIC PIO_LUSTRE_HINTS)
+  elseif (PIO_FILESYSTEM_HINTS STREQUAL gpfs)
+    message (STATUS "PIO will use gpfs filesystem hints")
+    target_compile_definitions (piof
+      PUBLIC PIO_GPFS_HINTS)
+  else ()
+    message (WARNING "${PIO_FILESYSTEM_HINTS} not valid option for PIO_FILESYSTEM_HINTS; use gpfs or lustre.")
+  endif ()
+endif ()
+
+#==============================================================================
+#  DEFINE THE INSTALL
+#==============================================================================
+
+# Library
+install (TARGETS piof DESTINATION lib)
+
+# Fortran Modules
+install (FILES ${PIO_Fortran_MODS} DESTINATION include)
+
+#==============================================================================
+#  DEFINE THE DEPENDENCIES
+#==============================================================================
+
+#===== pioc =====
+target_link_libraries(piof
+  PUBLIC pioc)
+
+#===== genf90 =====
+if (DEFINED GENF90_PATH)
+  add_custom_target(genf90
+    DEPENDS ${GENF90_PATH}/genf90.pl)
+else ()
+  ExternalProject_Add (genf90
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/genf90
+    GIT_REPOSITORY https://github.com/PARALLELIO/genf90
+    GIT_TAG genf90_140121
+    UPDATE_COMMAND ""
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    INSTALL_COMMAND "")
+  ExternalProject_Get_Property (genf90 SOURCE_DIR)
+  set (GENF90_PATH ${SOURCE_DIR})
+  unset (SOURCE_DIR)
+endif ()
+add_dependencies (piof genf90)
+
+#===== Fortran Source Generation with GenF90 =====
+foreach (SRC_FILE IN LISTS PIO_GenF90_SRCS)
+  add_custom_command (OUTPUT ${SRC_FILE}
+    COMMAND ${GENF90_PATH}/genf90.pl
+    ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in > ${SRC_FILE}
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in genf90)
+endforeach ()
+
+#===== MPI =====
+if (PIO_USE_MPISERIAL)
+  find_package (MPISERIAL COMPONENTS Fortran REQUIRED)
+  if (MPISERIAL_Fortran_FOUND)
+    target_compile_definitions (piof
+      PRIVATE _MPISERIAL)
+    target_include_directories (piof
+      PUBLIC ${MPISERIAL_Fortran_INCLUDE_DIRS})
+    target_link_libraries (piof
+      PUBLIC ${MPISERIAL_Fortran_LIBRARIES})
+
+    set (WITH_PNETCDF FALSE)
+    set (MPI_Fortran_INCLUDE_PATH ${MPISERIAL_Fortran_INCLUDE_DIRS})
+  endif ()
+else ()
+  find_package (MPI REQUIRED)
+endif ()
+
+# Check MPI I/O capabilities
+find_path(MPIF_H_PATH
+  NAMES mpif.h
+  PATHS ${MPI_Fortran_INCLUDE_PATH}
+  NO_DEFAULT_PATH)
+if (MPIF_H_PATH)
+  check_macro (MPI_HAS_MPIIO
+    NAME TryMPIIO.f90
+    HINTS ${CMAKE_MODULE_PATH}
+    DEFINITIONS -I{MPIF_H_PATH}
+    COMMENT "whether MPIIO is supported")
+  if (${MPI_HAS_MPIIO})
+    message (STATUS "MPIIO verified and enabled.")
+    target_compile_definitions (piof
+      PUBLIC USEMPIIO)
+  else ()
+    message (STATUS "MPIIO failed verification and therefore disabled.")
+  endif ()
+else ()
+  message (STATUS "MPIIO not detected and therefore disabled.")
+endif ()
+
+# Check for MPI Fortran module
+find_path(MPIMOD_PATH
+  NAMES mpi.mod MPI.mod
+  HINTS ${MPI_Fortran_INCLUDE_PATH})
+if (PIO_ENABLE_TIMING)
+  find_package (GPTL COMPONENTS Fortran_Perf QUIET)
+endif ()
+
+check_macro (MPI_HAS_Fortran_MOD
+  NAME TryMPIMod.f90
+  HINTS ${CMAKE_MODULE_PATH}
+  DEFINITIONS -I${MPIMOD_PATH}
+  COMMENT "whether MPI Fortran module is supported")
+if (${MPI_HAS_Fortran_MOD})
+  message (STATUS "MPI Fortran module verified and enabled.")
+else ()
+  message (STATUS "MPI Fortran module failed verification and therefore disabled.")
+  if (PIO_ENABLE_TIMING AND NOT GPTL_Fortran_Perf_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC NO_MPIMOD)
+  endif()
+endif ()
+
+#===== GPTL =====
+if (PIO_ENABLE_TIMING)
+  if (GPTL_Fortran_Perf_FOUND)
+    message (STATUS "Found GPTL Fortran Perf: ${GPTL_Fortran_Perf_LIBRARIES}")
+    target_include_directories (piof
+      PUBLIC ${GPTL_Fortran_INCLUDE_DIRS})
+    target_link_libraries (piof
+      PUBLIC ${GPTL_Fortran_LIBRARIES})
+  else ()
+    message (STATUS "Using internal GPTL Fortran library for timing")
+    target_link_libraries (piof
+      PUBLIC gptl)
+  endif ()
+endif ()
+
+#===== NetCDF-Fortran =====
+find_package (NetCDF "4.3.3" COMPONENTS Fortran)
+if (NetCDF_Fortran_FOUND)
+  target_include_directories (piof
+    PUBLIC ${NetCDF_Fortran_INCLUDE_DIRS})
+  target_compile_definitions (piof
+    PUBLIC _NETCDF)
+  target_link_libraries (piof
+    PUBLIC ${NetCDF_Fortran_LIBRARIES})
+  if (EXISTS ${NetCDF_Fortran_INCLUDE_DIR}/netcdf_par.h)
+    target_compile_definitions (piof
+      PUBLIC _NETCDF4)
+  endif ()
+else ()
+  target_compile_definitions (piof
+    PUBLIC _NONETCDF)
+endif ()
+
+#===== PnetCDF =====
+if (WITH_PNETCDF)
+  find_package (PnetCDF "1.6" COMPONENTS Fortran REQUIRED)
+endif ()
+if (PnetCDF_Fortran_FOUND)
+  target_include_directories (piof
+    PUBLIC ${PnetCDF_Fortran_INCLUDE_DIRS})
+  target_compile_definitions (piof
+    PUBLIC _PNETCDF)
+  target_link_libraries (piof
+    PUBLIC ${PnetCDF_Fortran_LIBRARIES})
+
+  # Check library for varn functions
+  set (CMAKE_REQUIRED_LIBRARIES ${PnetCDF_Fortran_LIBRARY})
+  check_function_exists (ncmpi_get_varn PnetCDF_Fortran_HAS_VARN)
+  if (PnetCDF_Fortran_HAS_VARN)
+    target_compile_definitions(piof
+      PUBLIC USE_PNETCDF_VARN
+      PUBLIC USE_PNETCDF_VARN_ON_READ)
+  endif()
+else ()
+  target_compile_definitions (piof
+    PUBLIC _NOPNETCDF)
+endif ()
+
+#===== Add EXTRAs =====
+target_include_directories (piof
+  PUBLIC ${PIO_Fortran_EXTRA_INCLUDE_DIRS})
+target_link_libraries (piof
+  PUBLIC ${PIO_Fortran_EXTRA_LIBRARIES})
+target_compile_options (piof
+  PRIVATE ${PIO_Fortran_EXTRA_COMPILE_OPTIONS})
+target_compile_definitions (piof
+  PUBLIC ${PIO_Fortran_EXTRA_COMPILE_DEFINITIONS})
+if (PIO_Fortran_EXTRA_LINK_FLAGS)
+  set_target_properties(piof PROPERTIES
+    LINK_FLAGS ${PIO_Fortran_EXTRA_LINK_FLAGS})
+endif ()
+
+#===== Check for necessities =====
+if (NOT PnetCDF_Fortran_FOUND AND NOT NetCDF_Fortran_FOUND)
+  message (FATAL_ERROR "Must have PnetCDF and/or NetCDF Fortran libraries")
+endif ()
diff --git a/externals/pio2/src/flib/pio.F90 b/src/externals/pio2/src/flib/pio.F90
similarity index 90%
rename from externals/pio2/src/flib/pio.F90
rename to src/externals/pio2/src/flib/pio.F90
index 0609c506c53..cf8c96105b0 100644
--- a/externals/pio2/src/flib/pio.F90
+++ b/src/externals/pio2/src/flib/pio.F90
@@ -1,7 +1,7 @@
 !>
-!! @file
+!! @file 
 !! @brief User interface Module for PIO, this is the only file a user program should 'use'
-!!
+!! 
 !<
 
 module pio
@@ -16,9 +16,14 @@ module pio
        pio_seterrorhandling, pio_setframe, pio_init, pio_get_local_array_size, &
        pio_freedecomp, pio_syncfile, &
        pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, &
-       PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available
+       PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available, &
+       pio_set_rearr_opts
 
   use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, &
+       pio_rearr_opt_t, pio_rearr_comm_fc_opt_t, pio_rearr_comm_fc_2d_enable,&
+       pio_rearr_comm_fc_1d_comp2io, pio_rearr_comm_fc_1d_io2comp,&
+       pio_rearr_comm_fc_2d_disable, pio_rearr_comm_unlimited_pend_req,&
+       pio_rearr_comm_p2p, pio_rearr_comm_coll,&
        pio_int, pio_real, pio_double, pio_noerr, iotype_netcdf, &
        iotype_pnetcdf,  pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
        pio_iotype_pnetcdf,pio_iotype_netcdf, &
@@ -28,9 +33,9 @@ module pio
        pio_nofill, pio_unlimited, pio_fill_int, pio_fill_double, pio_fill_float, &
 #endif
        pio_64bit_offset, pio_64bit_data, &
-       pio_internal_error, pio_bcast_error, pio_return_error, pio_rearr_opt_t
+       pio_internal_error, pio_bcast_error, pio_return_error, pio_default
 
-  use piodarray, only : pio_read_darray, pio_write_darray, pio_set_buffer_size_limit
+  use piodarray, only : pio_read_darray, pio_write_darray, pio_set_buffer_size_limit  
 
   use pio_nf, only:        &
        PIO_enddef,            &
@@ -53,12 +58,14 @@ module pio
        PIO_def_var   ,        &
        PIO_def_var_deflate   ,        &
        PIO_redef     ,          &
+       PIO_set_log_level,          &
        PIO_inquire_variable , &
        PIO_inquire_dimension, &
        PIO_set_chunk_cache, &
        PIO_get_chunk_cache, &
        PIO_set_var_chunk_cache, &
-       PIO_get_var_chunk_cache
+       PIO_get_var_chunk_cache, &
+       PIO_strerror
 
   use pionfatt_mod, only : PIO_put_att   => put_att,        &
        PIO_get_att   => get_att
@@ -111,11 +118,11 @@ integer(C_INT) function PIOc_iam_iotask(iosysid, iotask) &
          logical(C_BOOL), intent(out) :: iotask
        end function PIOc_iam_iotask
     end interface
-
+    
     ierr = PIOc_iam_iotask(iosystem%iosysid, ctask)
     task = ctask
   end function pio_iam_iotask
-
+  
 !>
 !! @public
 !! @brief Integer function returns rank of IO task.
@@ -131,7 +138,7 @@ integer(C_INT) function PIOc_iotask_rank(iosysid, rank) &
          integer(C_INT), intent(out) :: rank
        end function PIOc_iotask_rank
     end interface
-
+    
     ierr = PIOc_iotask_rank(iosystem%iosysid, rank)
   end function pio_iotask_rank
 
diff --git a/externals/pio2/src/flib/pio_kinds.F90 b/src/externals/pio2/src/flib/pio_kinds.F90
similarity index 100%
rename from externals/pio2/src/flib/pio_kinds.F90
rename to src/externals/pio2/src/flib/pio_kinds.F90
diff --git a/externals/pio2/src/flib/pio_nf.F90 b/src/externals/pio2/src/flib/pio_nf.F90
similarity index 94%
rename from externals/pio2/src/flib/pio_nf.F90
rename to src/externals/pio2/src/flib/pio_nf.F90
index 5c798f834ec..1b19beec4b9 100644
--- a/externals/pio2/src/flib/pio_nf.F90
+++ b/src/externals/pio2/src/flib/pio_nf.F90
@@ -14,7 +14,7 @@ module pio_nf
        pio_def_var_deflate                                  , &
        pio_def_var_chunking                                 , &
        pio_def_dim                                          , &
-       pio_inq_attname                                      , & 
+       pio_inq_attname                                      , &
        pio_inq_att                                          , &
        pio_inq_attlen                                       , &
        pio_inq_varid                                        , &
@@ -23,7 +23,7 @@ module pio_nf
        pio_inq_varndims                                     , &
        pio_inq_vardimid                                     , &
        pio_inq_varnatts                                     , &
-       pio_inq_var_deflate                                  , &       
+       pio_inq_var_deflate                                  , &
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
@@ -36,7 +36,9 @@ module pio_nf
        pio_get_chunk_cache                                  , &
        pio_set_var_chunk_cache                              , &
        pio_get_var_chunk_cache                              , &
-       pio_redef
+       pio_redef                                            , &
+       pio_set_log_level                                    , &
+       pio_strerror
 !       pio_copy_att    to be done
 
   interface pio_def_var
@@ -44,11 +46,11 @@ module pio_nf
           def_var_0d_desc                                   , &
           def_var_md_desc                                   , &
           def_var_0d_id                                     , &
-          def_var_md_id 
+          def_var_md_id
   end interface
   interface pio_def_var_deflate
      module procedure &
-          def_var_deflate_desc                              , &         
+          def_var_deflate_desc                              , &
           def_var_deflate_id
   end interface
   interface pio_def_var_chunking
@@ -180,13 +182,24 @@ module pio_nf
      module procedure &
           enddef_desc                                       , &
           enddef_id
-  end interface
+  end interface pio_enddef
+
   interface pio_redef
      module procedure &
           redef_desc                                        , &
           redef_id
   end interface
 
+  interface pio_set_log_level
+     module procedure &
+          set_log_level
+  end interface pio_set_log_level
+
+  interface pio_strerror
+     module procedure &
+          strerror
+  end interface pio_strerror
+
   interface pio_inquire
      module procedure &
           inquire_desc                                      , &
@@ -221,7 +234,7 @@ module pio_nf
 !! @defgroup PIO_inq_dimid PIO_inq_dimid
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimid
 !! @brief Returns the netcdf dimension id for the name.
 !! @details
@@ -239,7 +252,7 @@ integer function inq_dimid_desc(File                      ,name,dimid) result(ie
     ierr = inq_dimid_id(file%fh                             ,name,dimid)
   end function inq_dimid_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimid
 !! @brief Returns the netcdf dimension id for the name.
 !<
@@ -264,9 +277,9 @@ end function inq_dimid_id
 !! @defgroup PIO_inquire_dimension PIO_inquire_dimension
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_dimension
-!! @brief  Get information about a particular dimension in netcdf file 
+!! @brief  Get information about a particular dimension in netcdf file
 !! @details
 !! @param ncid : A netcdf file descriptor returned by \ref PIO_openfile or \ref PIO_createfile.
 !! @param dimid : The netcdf dimension ID.
@@ -282,9 +295,9 @@ integer function inquire_dimension_desc(file              , dimid, name, len) re
     ierr = Inquire_dimension_id(file%fh                     , dimid, name, len)
   end function inquire_dimension_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_dimension
-!! @brief  Get information about a particular dimension in netcdf file 
+!! @brief  Get information about a particular dimension in netcdf file
 !<
   integer function inquire_dimension_id(ncid                , dimid, name, len) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -305,9 +318,9 @@ end function inquire_dimension_id
 !! @defgroup PIO_inq_dimlen PIO_inq_dimlen
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param dimid : The netcdf dimension ID.
@@ -321,9 +334,9 @@ integer function inq_dimlen_desc(File                     , dimid, len) result(i
     ierr = inq_dimlen_id(file%fh                            ,dimid,len)
   end function inq_dimlen_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !<
   integer function inq_dimlen_desc_long(File                , dimid, len) result(ierr)
     type(file_desc_t)                                       , intent(in) :: File
@@ -332,9 +345,9 @@ integer function inq_dimlen_desc_long(File                , dimid, len) result(i
     ierr = inq_dimlen_id_long(file%fh                       ,dimid,len)
   end function inq_dimlen_desc_long
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !<
   integer function inq_dimlen_id(ncid                       , dimid, len) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -345,9 +358,9 @@ integer function inq_dimlen_id(ncid                       , dimid, len) result(i
     len = int(llen)
   end function inq_dimlen_id
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !<
   integer function inq_dimlen_id_long(ncid                  , dimid, len) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -362,7 +375,7 @@ integer(C_INT) function PIOc_inq_dimlen(ncid         ,dimid,len) &
          integer(c_size_t) :: len
        end function PIOc_inq_dimlen
     end interface
-    
+
     ierr = PIOc_inq_dimlen(ncid                             ,dimid-1,len)
   end function inq_dimlen_id_long
 
@@ -371,9 +384,9 @@ end function inq_dimlen_id_long
 !! @defgroup PIO_inq_dimname PIO_inq_dimname
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimname
-!! @brief  Get information about the name of of a dimension. 
+!! @brief  Get information about the name of of a dimension.
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param dimid : The netcdf dimension ID.
@@ -387,9 +400,9 @@ integer function inq_dimname_desc(File                    , dimid, name) result(
     ierr = inq_dimname_id(file%fh                           ,dimid,name)
   end function inq_dimname_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimname
-!! @brief  Get information about the name of of a dimension. 
+!! @brief  Get information about the name of of a dimension.
 !<
   integer function inq_dimname_id(ncid                      , dimid, name) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -416,7 +429,7 @@ end function inq_dimname_id
 !! @defgroup PIO_inq_ndims PIO_inq_ndims
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_ndims
 !! @brief  Get information about the number of dimensions of a file or group.
 !! @details
@@ -430,7 +443,7 @@ integer function inq_ndims_desc(File                      , ndims) result(ierr)
     ierr = inq_ndims_id(file%fh                             , ndims)
   end function inq_ndims_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_ndims
 !! @brief  Get information about the number of dimensions of a file or group.
 !<
@@ -452,7 +465,7 @@ end function inq_ndims_id
 !! @defgroup PIO_inq_nvars PIO_inq_nvars
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_nvars
 !! @brief  Get information about the number of variables in a file or group.
 !! @details
@@ -466,7 +479,7 @@ integer function inq_nvars_desc(File                      , nvars) result(ierr)
     ierr = inq_nvars_id(file%fh                             , nvars)
   end function inq_nvars_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_nvars
 !! @brief  Get information about the number of variables in a file or group.
 !<
@@ -488,7 +501,7 @@ end function inq_nvars_id
 !! @defgroup PIO_inq_natts PIO_inq_natts
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_natts
 !! @brief  Get information about the number of global attributes in a file or group.
 !! @details
@@ -502,7 +515,7 @@ integer function inq_natts_desc(File                      , natts) result(ierr)
     ierr = inq_natts_id(file%fh                             , natts)
   end function inq_natts_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_natts
 !! @brief  Get information about the number of global attributes in a file or group.
 !<
@@ -524,7 +537,7 @@ end function inq_natts_id
 !! @defgroup PIO_inq_unlimdim PIO_inq_unlimdim
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_unlimdm
 !! @brief  Get information about the unlimited dimension in a file.
 !! @details
@@ -538,7 +551,7 @@ integer function inq_unlimdim_desc(File                   , unlimdim) result(ier
     ierr = inq_unlimdim_id(file%fh                          , unlimdim)
   end function inq_unlimdim_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_unlimdm
 !! @brief  Get information about the unlimited dimension in a file.
 !<
@@ -561,13 +574,13 @@ end function inq_unlimdim_id
 !! @defgroup PIO_inquire PIO_inquire
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire
 !! @brief Gets metadata information for netcdf file.
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param nDimensions :  Number of dimensions defined for the netcdf file
-!! @param nVariables : Number of variables defined for the netcdf file 
+!! @param nVariables : Number of variables defined for the netcdf file
 !! @param nAttributes : Number of attributes defined for the netcdf file
 !! @param unlimitedDimID : the Unlimited dimension ID
 !! @retval ierr @copydoc error_return
@@ -580,11 +593,11 @@ integer function inquire_desc(File                        ,nDimensions,nVariable
          nVariables                                         ,   &! number of variables
          nAttributes                                        ,  & ! number of global attributes
          unlimitedDimID ! ID of unlimited dimension
-    
+
     ierr = inquire_id(file%fh                               ,ndimensions,nvariables,nattributes,unlimitedDimID)
   end function inquire_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -605,7 +618,7 @@ end function inquire_id
 !>
 !! @defgroup PIO_enddef PIO_enddef
 !<
-!> 
+!>
 !! @public
 !! @ingroup PIO_enddef
 !! @brief Exits netcdf define mode.
@@ -617,7 +630,7 @@ integer function enddef_desc(File) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     ierr = enddef_id(file%fh)
   end function enddef_desc
-!> 
+!>
 !! @public
 !! @ingroup PIO_enddef
 !! @brief Wrapper for the C function \ref PIOc_enddef .
@@ -636,7 +649,7 @@ end function enddef_id
 !>
 !! @defgroup PIO_redef PIO_redef
 !<
-!> 
+!>
 !! @public
 !! @ingroup PIO_redef
 !! @brief Exits netcdf define mode.
@@ -648,18 +661,69 @@ integer function redef_desc(File) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     ierr = redef_id(file%fh)
   end function redef_desc
-!> 
+
+!>
+!! @defgroup PIO_set_log_level
+!<
+!>
+!! @ingroup PIO_set_log_level
+!! Sets the logging level. Only takes effect if PIO was built with
+!! PIO_ENABLE_LOGGING=On
+!!
+!! @param log_level the logging level.
+!! @retval ierr @copydoc error_return
+!<
+  integer function set_log_level(log_level) result(ierr)
+    integer, intent(in) :: log_level
+    interface
+       integer(C_INT) function PIOc_set_log_level(log_level) &
+            bind(C, name="PIOc_set_log_level")
+         use iso_c_binding
+         integer(C_INT), value :: log_level
+       end function PIOc_set_log_level
+    end interface
+    ierr = PIOc_set_log_level(log_level)
+  end function set_log_level
+
+  !>
+  !! @defgroup PIO_strerror
+  !<
+  !>
+  !! @ingroup PIO_strerror
+  !! Returns a descriptive string for an error code.
+  !!
+  !! @param errcode the error code
+  !! @retval a description of the error
+  !<
+  integer function strerror(errcode, errmsg) result(ierr)
+    integer, intent(in) :: errcode
+    character(len=*), intent(out) :: errmsg
+    interface
+       integer(C_INT) function PIOc_strerror(errcode, errmsg) &
+            bind(C, name="PIOc_strerror")
+         use iso_c_binding
+         integer(C_INT), value :: errcode
+         character(C_CHAR) :: errmsg(*)
+       end function PIOc_strerror
+    end interface
+    errmsg = C_NULL_CHAR
+    ierr = PIOc_strerror(errcode, errmsg)
+    call replace_c_null(errmsg)
+
+  end function strerror
+
+!>
 !! @public
 !! @ingroup PIO_redef
 !! @brief Wrapper for the C function \ref PIOc_redef .
 !<
   integer function redef_id(ncid) result(ierr)
-    integer                                                 ,intent(in) :: ncid
+    integer, intent(in) :: ncid
     interface
        integer(C_INT) function PIOc_redef(ncid) &
-            bind(C                                          ,name="PIOc_redef")
+            bind(C, name="PIOc_redef")
          use iso_c_binding
-         integer(C_INT)                                     , value :: ncid
+         integer(C_INT), value :: ncid
        end function PIOc_redef
     end interface
     ierr = PIOc_redef(ncid)
@@ -669,7 +733,7 @@ end function redef_id
 !! @defgroup PIO_def_dim PIO_def_dim
 !! @brief A set of functions to define dimensions and their attributes in NetCDF files.
 !<
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief Defines the netcdf dimension.
@@ -688,7 +752,7 @@ integer function def_dim_int_desc(File                    ,name,len,dimid) resul
 
     ierr = def_dim_id(file%fh                               ,name,int(len,pio_offset_kind),dimid)
   end function def_dim_int_desc
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief  Defines the netcdf dimension.
@@ -701,7 +765,7 @@ integer function def_dim_int_id(ncid                      ,name,len,dimid) resul
 
     ierr = def_dim_id(ncid                                  ,name,int(len,pio_offset_kind),dimid)
   end function def_dim_int_id
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief  Defines the netcdf dimension.
@@ -715,7 +779,7 @@ integer function def_dim_desc(File                        ,name,len,dimid) resul
 
     ierr = def_dim_id(file%fh                               ,name,len,dimid)
   end function def_dim_desc
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief  Defines the netcdf dimension.
@@ -745,9 +809,9 @@ end function def_dim_id
 !! @defgroup PIO_inquire_variable PIO_inquire_variable
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes  
+!! @brief Inquires if a NetCDF variable is present and returns its attributes
 !! @details
 !! @param ncid : A netcdf file descriptor returned by \ref PIO_openfile or \ref PIO_createfile.
 !! @param vardesc @copydoc var_desc_t
@@ -769,9 +833,9 @@ integer function inquire_variable_desc(file               , vardesc, name, xtype
     ierr = pio_inquire_variable(file%fh                     ,vardesc%varid,name,xtype,ndims,dimids,natts)
   end function inquire_variable_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes  
+!! @brief Inquires if a NetCDF variable is present and returns its attributes
 !<
   integer function inquire_variable_vid(file                , varid, name, xtype, ndims, dimids, natts) result(ierr)
     type(file_desc_t)                                       ,               intent(in) :: file
@@ -784,9 +848,9 @@ integer function inquire_variable_vid(file                , varid, name, xtype,
     ierr = pio_inquire_variable(file%fh                     ,varid,name,xtype,ndims,dimids,natts)
   end function inquire_variable_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes  
+!! @brief Inquires if a NetCDF variable is present and returns its attributes
 !<
   integer function inquire_variable_id(ncid                 , varid, name, xtype, ndims, dimids, natts) result(ierr)
     integer                                                 ,                intent( in) :: ncid
@@ -807,7 +871,7 @@ end function inquire_variable_id
 !! @defgroup PIO_inq_vardimid PIO_inq_vardimid
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vardimid
 !! @brief returns the dimids of the variable as an interger array
 !! @details
@@ -826,7 +890,7 @@ integer function inq_vardimid_desc(File                   ,vardesc,dimids) resul
     ierr = pio_inq_vardimid(File%fh                         , vardesc%varid, dimids)
   end function inq_vardimid_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vardimid
 !! @brief returns the dimids of the variable as an interger array
 !<
@@ -840,10 +904,10 @@ integer function inq_vardimid_vid(File                    ,varid,dimids) result(
     ierr = pio_inq_vardimid(File%fh                         , varid, dimids)
   end function inq_vardimid_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vardimid
 !! @brief returns the dimids of the variable as an interger array
-!<    
+!<
   integer function inq_vardimid_id(ncid                     ,varid,dimids) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -859,7 +923,7 @@ integer(C_INT) function PIOc_inq_vardimid(ncid       ,varid,dimids) &
        end function PIOc_inq_vardimid
     end interface
     integer :: i                                            , ndims
-    
+
     ierr = inq_varndims_id(ncid                             ,varid,ndims)
     allocate(cdimids(ndims))
 
@@ -875,13 +939,13 @@ end function inq_vardimid_id
 !! @defgroup PIO_inq_varndims PIO_inq_varndims
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varndims
 !! @brief Gets the number of dimension associated with a netcdf variable
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param ndims : The number of dimensions for the variable 
+!! @param ndims : The number of dimensions for the variable
 !! @retval ierr @copydoc error_return
 !<
   integer function inq_varndims_desc(File                   ,vardesc,ndims) result(ierr)
@@ -893,7 +957,7 @@ integer function inq_varndims_desc(File                   ,vardesc,ndims) result
     ierr = pio_inq_varndims(File%fh                         , vardesc%varid, ndims)
   end function inq_varndims_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varndims
 !! @brief Gets the number of dimension associated with a netcdf variable
 !<
@@ -906,7 +970,7 @@ integer function inq_varndims_vid(File                    ,varid,ndims) result(i
     ierr = pio_inq_varndims(File%fh                         , varid, ndims)
   end function inq_varndims_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varndims
 !! @brief Gets the number of dimension associated with a netcdf variable
 !<
@@ -930,7 +994,7 @@ end function inq_varndims_id
 !! @defgroup PIO_inq_vartype PIO_inq_vartype
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vartype
 !! @brief Gets metadata information for netcdf file.
 !! @details
@@ -948,7 +1012,7 @@ integer function inq_vartype_desc(File                    ,vardesc,type) result(
     ierr = pio_inq_vartype(File%fh                          , vardesc%varid, type)
   end function inq_vartype_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vartype
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -961,7 +1025,7 @@ integer function inq_vartype_vid(File                     ,varid,type) result(ie
     ierr = pio_inq_vartype(File%fh                          , varid, type)
   end function inq_vartype_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vartype
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -987,7 +1051,7 @@ end function inq_vartype_id
 !!  @defgroup PIO_inq_varnatts PIO_inq_varnatts
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varnatts
 !! @brief Gets metadata information for netcdf file.
 !! @details
@@ -1005,7 +1069,7 @@ integer function inq_varnatts_desc(File                   ,vardesc,natts) result
     ierr = pio_inq_varnatts(File%fh                         , vardesc%varid,natts)
   end function inq_varnatts_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varnatts
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1018,7 +1082,7 @@ integer function inq_varnatts_vid(File                    ,varid,natts) result(i
     ierr = pio_inq_varnatts(File%fh                         , varid, natts)
   end function inq_varnatts_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varnatts
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1044,7 +1108,7 @@ end function inq_varnatts_id
 !!  @defgroup PIO_inq_var_deflate PIO_inq_var_deflate
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !! @details
@@ -1066,7 +1130,7 @@ integer function inq_var_deflate_desc(File, vardesc, shuffle, deflate, &
   end function inq_var_deflate_desc
 
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1082,7 +1146,7 @@ integer function inq_var_deflate_vid(File, varid, shuffle, deflate, deflate_leve
   end function inq_var_deflate_vid
 
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1108,12 +1172,12 @@ end function PIOc_inq_var_deflate
 
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
   end function inq_var_deflate_id
-  
+
 !>
 !! @defgroup PIO_inq_varname
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varname
 !! @brief Get the name associated with a variable
 !! @details
@@ -1127,12 +1191,12 @@ integer function inq_varname_desc(File                    ,vardesc,name) result(
     type (File_desc_t)                                      , intent(in)   :: File
     type (Var_desc_t)                                       , intent(in)    :: vardesc
     character(len=*)                                        , intent(out)    :: name
-    
+
     ierr = pio_inq_varname(file%fh                          ,vardesc%varid,name)
 
   end function inq_varname_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varname
 !! @brief Get the name associated with a variable
 !<
@@ -1141,12 +1205,12 @@ integer function inq_varname_vid(File                     ,varid,name) result(ie
     type (File_desc_t)                                      , intent(in)   :: File
     integer                                                 , intent(in)    :: varid
     character(len=*)                                        , intent(out)    :: name
-    
+
     ierr = pio_inq_varname(file%fh                          ,varid,name)
 
   end function inq_varname_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varname
 !! @brief Get the name associated with a variable
 !<
@@ -1173,10 +1237,10 @@ end function inq_varname_id
 !>
 !! @defgroup PIO_inq_varid
 !<
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name 
+!! @brief Returns the ID of a netcdf variable given its name
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param name   : Name of the returned attribute
@@ -1191,10 +1255,10 @@ integer function inq_varid_desc(File,name,vardesc) result(ierr)
 
     ierr = pio_inq_varid(File%fh, name, vardesc%varid)
   end function inq_varid_desc
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name 
+!! @brief Returns the ID of a netcdf variable given its name
 !<
   integer function inq_varid_vid(File,name,varid) result(ierr)
 
@@ -1204,10 +1268,10 @@ integer function inq_varid_vid(File,name,varid) result(ierr)
 
     ierr = pio_inq_varid(File%fh, name, varid)
   end function inq_varid_vid
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name 
+!! @brief Returns the ID of a netcdf variable given its name
 !<
   integer function inq_varid_id(ncid,name,varid) result(ierr)
 
@@ -1234,9 +1298,9 @@ end function inq_varid_id
 !! @defgroup PIO_inq_attlen
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length 
+!! @brief  Gets the attribute length
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
@@ -1255,9 +1319,9 @@ integer function inq_attlen_desc(File,vardesc,name,len) result(ierr)
 
   end function inq_attlen_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length 
+!! @brief  Gets the attribute length
 !<
   integer function inq_attlen_vid(File,varid,name,len) result(ierr)
 
@@ -1270,9 +1334,9 @@ integer function inq_attlen_vid(File,varid,name,len) result(ierr)
 
   end function inq_attlen_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length 
+!! @brief  Gets the attribute length
 !<
   integer function inq_attlen_id(ncid,varid,name,len) result(ierr)
     integer, intent(in) :: ncid
@@ -1298,15 +1362,15 @@ end function inq_attlen_id
 !! @defgroup PIO_inq_att PIO_inq_att
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_att
 !! @brief  Gets information about attributes
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param name : Name of the attribute 
+!! @param name : Name of the attribute
 !! @param xtype : The type of attribute
-!! @param len : The length of the attribute 
+!! @param len : The length of the attribute
 !! @retval ierr @copydoc error_return
 !<
   integer function inq_att_desc(File,vardesc,name,xtype,len) result(ierr)
@@ -1321,7 +1385,7 @@ integer function inq_att_desc(File,vardesc,name,xtype,len) result(ierr)
 
   end function inq_att_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_att
 !! @brief  Gets information about attributes
 !<
@@ -1337,7 +1401,7 @@ integer function inq_att_vid(File,varid,name,xtype,len) result(ierr)
 
   end function inq_att_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_att
 !! @brief  Gets information about attributes
 !<
@@ -1363,17 +1427,17 @@ integer(C_INT) function PIOc_inq_att(ncid,varid,name,xtype,len) &
          integer(C_SIZE_T) :: len
        end function PIOc_inq_att
     end interface
-    
+
     ierr = PIOc_inq_att(ncid,varid-1,trim(name)//C_NULL_CHAR,ixtype,xlen)
     if(present(len)) len=xlen
     if(present(xtype)) xtype = ixtype
 
   end function inq_att_id
-!> 
+!>
 !! @defgroup PIO_inq_attname
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attname
 !! @brief  Gets the name of an attribute
 !<
@@ -1387,7 +1451,7 @@ integer function inq_attname_desc(File,vdesc,attnum,name) result(ierr)
 
   end function inq_attname_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attname
 !! @brief   Gets the name of an attribute
 !<
@@ -1396,12 +1460,12 @@ integer function inq_attname_vid(File,varid,attnum,name) result(ierr)
     integer, intent(in)           :: varid
     integer, intent(in)              :: attnum !Attribute number
     character(len=*), intent(out)     :: name
-    
+
     ierr = inq_attname_id(file%fh,varid,attnum,name)
 
   end function inq_attname_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attname
 !! @brief   Gets the name of an attribute
 !<
@@ -1432,14 +1496,14 @@ end function inq_attname_id
 !! @defgroup PIO_def_var PIO_def_var
 !<
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var
 !! @brief Defines a netcdf variable
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param name : The name of the variable to define
-!! @param type : The type of variable 
+!! @param type : The type of variable
 !! @param vardesc @copydoc var_desc_t
 !! @retval ierr @copydoc error_return
 !<
@@ -1454,8 +1518,8 @@ integer function def_var_0d_desc(File,name,type,vardesc) result(ierr)
     ierr = def_var_md_id(File%fh,name,type,dimids,vardesc%varid)
 
   end function def_var_0d_desc
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var
 !! @brief Defines a netcdf variable
 !<
@@ -1471,14 +1535,14 @@ integer function def_var_0d_id(ncid,name,type,varid) result(ierr)
 
   end function def_var_0d_id
 
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_var
 !! @brief Defines the a netcdf variable
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param name : The name of the variable to define
-!! @param type : The type of variable 
+!! @param type : The type of variable
 !! @param dimids : The dimension identifier returned by \ref PIO_def_dim
 !! @param vardesc @copydoc var_desc_t
 !! @retval ierr @copydoc error_return
@@ -1492,8 +1556,8 @@ integer function def_var_md_desc(File,name,type,dimids,vardesc) result(ierr)
 
     ierr = def_var_md_id(file%fh,name,type,dimids,vardesc%varid)
   end function def_var_md_desc
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var
 !! @brief Defines a netcdf variable
 !<
@@ -1526,8 +1590,8 @@ end function PIOc_def_var
     varid = varid+1
   end function def_var_md_id
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
@@ -1554,8 +1618,8 @@ end function PIOc_def_var_deflate
     ierr = PIOc_def_var_deflate(file%fh, varid-1, shuffle, deflate, deflate_level)
   end function def_var_deflate_id
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
@@ -1570,8 +1634,8 @@ integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_l
     ierr = def_var_deflate_id(file, vardesc%varid, shuffle, deflate, deflate_level)
   end function def_var_deflate_desc
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var_chunking
 !! @brief Changes chunking settings for a netCDF-4/HDF5 variable.
 !<
@@ -1581,7 +1645,7 @@ integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ier
     integer, intent(in) :: storage
     integer, intent(in) :: chunksizes(:)
     integer(C_INT) :: cchunksizes(PIO_MAX_VAR_DIMS)
-    integer :: ndims, i    
+    integer :: ndims, i
 
     interface
        integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes) &
@@ -1601,15 +1665,15 @@ end function PIOc_def_var_chunking
     ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
   end function def_var_chunking
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_set_chunk_cache
 !! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
 !<
   integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
        chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
-    integer, intent(in) :: iotype 
+    integer, intent(in) :: iotype
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
@@ -1631,9 +1695,9 @@ end function PIOc_set_chunk_cache
          chunk_cache_preemption)
   end function set_chunk_cache
 
-!> 
+!>
 !! @public
-!! @ingroup PIO_set_chunk_cache  
+!! @ingroup PIO_get_chunk_cache
 !! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
 !<
   integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
@@ -1661,9 +1725,9 @@ end function PIOc_get_chunk_cache
          chunk_cache_preemption)
   end function get_chunk_cache
 
-!> 
-!! @public 
-!! @ingroup PIO_set_chunk_cache
+!>
+!! @public
+!! @ingroup PIO_set_var_chunk_cache
 !! @brief Changes chunk cache settings for a variable in a netCDF-4/HDF5 file.
 !<
   integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, &
@@ -1691,8 +1755,8 @@ end function PIOc_set_var_chunk_cache
          chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_id
 
-  !> 
-!! @public 
+  !>
+!! @public
 !! @ingroup PIO_set_var_chunk_cache
 !! @brief Changes chunk cacne for a variable.
 !<
@@ -1708,8 +1772,8 @@ integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_desc
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_get_var_chunk_cache
 !! @brief Get the chunk cache settings for a variable.
 !<
@@ -1725,8 +1789,8 @@ integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function get_var_chunk_cache_desc
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_get_var_chunk_cache
 !! @brief Get the chunk cache settings for a variable.
 !<
diff --git a/externals/pio2/src/flib/pio_support.F90 b/src/externals/pio2/src/flib/pio_support.F90
similarity index 100%
rename from externals/pio2/src/flib/pio_support.F90
rename to src/externals/pio2/src/flib/pio_support.F90
diff --git a/externals/pio2/src/flib/pio_types.F90 b/src/externals/pio2/src/flib/pio_types.F90
similarity index 76%
rename from externals/pio2/src/flib/pio_types.F90
rename to src/externals/pio2/src/flib/pio_types.F90
index c3482ffbb26..d8bb4a3f019 100644
--- a/externals/pio2/src/flib/pio_types.F90
+++ b/src/externals/pio2/src/flib/pio_types.F90
@@ -13,7 +13,7 @@ module pio_types
     !-------------------------------------------
     type, public :: DecompMap_t
 #ifdef SEQUENCE
-	sequence
+        sequence
 #endif
         integer(i4) :: start
         integer(i4) :: length
@@ -73,7 +73,7 @@ module pio_types
 !<
    type, public :: io_desc_t
 #ifdef SEQUENCE
-	sequence
+        sequence
 #endif
         integer(i4)         :: ioid
     end type
@@ -140,6 +140,14 @@ module pio_types
   integer(i4), public, parameter :: PIO_BCAST_ERROR = -52
   integer(i4), public, parameter :: PIO_RETURN_ERROR = -53
 
+!>
+!! @public
+!! @defgroup PIO_error_method error_methods
+!! @details
+!! Use this instead of ios to set error handling for the library.
+!<
+  integer(i4), public, parameter :: PIO_DEFAULT = -1
+
 !>
 !! @public
 !! @defgroup error_return error return codes
@@ -229,17 +237,69 @@ module pio_types
 #endif
    integer, public, parameter :: PIO_num_OST =  16
 
-   type, public :: PIO_rearr_comm_fc_opt_t
-      logical :: enable_hs            ! Enable handshake?
-      logical :: enable_isend         ! Enable isends?
-      integer :: max_pend_req         ! Maximum pending requests
+!>
+!! @defgroup PIO_rearr_comm_t PIO_rearr_comm_t
+!! @public 
+!! @brief The two choices for rearranger communication
+!! @details
+!!  - PIO_rearr_comm_p2p : Point to point
+!!  - PIO_rearr_comm_coll : Collective
+!>
+    enum, bind(c)
+      enumerator :: PIO_rearr_comm_p2p = 0
+      enumerator :: PIO_rearr_comm_coll
+    end enum
+
+!>
+!! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir
+!! @public 
+!! @brief The four choices for rearranger communication direction
+!! @details
+!!  - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa
+!!  - PIO_rearr_comm_fc_1d_comp2io: COMM procs to IO procs only
+!!  - PIO_rearr_comm_fc_1d_io2comp: IO procs to COMM procs only
+!!  - PIO_rearr_comm_fc_2d_disable: Disable flow control
+!>
+    enum, bind(c)
+      enumerator :: PIO_rearr_comm_fc_2d_enable = 0
+      enumerator :: PIO_rearr_comm_fc_1d_comp2io
+      enumerator :: PIO_rearr_comm_fc_1d_io2comp
+      enumerator :: PIO_rearr_comm_fc_2d_disable
+    end enum
+
+!>
+!! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options
+!! @brief Type that defines the PIO rearranger options
+!! @details
+!!  - enable_hs : Enable handshake (true/false) 
+!!  - enable_isend : Enable Isends (true/false)
+!!  - max_pend_req : Maximum pending requests (To indicated unlimited
+!!                    number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
+!>
+   type, bind(c), public :: PIO_rearr_comm_fc_opt_t
+      logical(c_bool) :: enable_hs            ! Enable handshake?
+      logical(c_bool) :: enable_isend         ! Enable isends?
+      integer(c_int) :: max_pend_req         ! Maximum pending requests
     end type PIO_rearr_comm_fc_opt_t
 
-    type, public :: PIO_rearr_opt_t
-      integer                         :: comm_type
-      integer                         :: fcd       ! Flow control direction
+    integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1
+!>
+!! @defgroup PIO_rearr_options PIO_rearr_options
+!! @brief Type that defines the PIO rearranger options
+!! @details
+!!  - comm_type : @copydoc PIO_rearr_comm_t
+!!  - fcd : @copydoc PIO_rearr_comm_dir
+!!  - comm_fc_opts : @copydoc PIO_rearr_comm_fc_options
+!>
+    type, bind(c), public :: PIO_rearr_opt_t
+      integer(c_int)                         :: comm_type
+      integer(c_int)                         :: fcd       ! Flow control direction
       type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io
       type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp
     end type PIO_rearr_opt_t
 
+    public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
+              PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
+              PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable
+
 end module pio_types
diff --git a/externals/pio2/src/flib/piodarray.F90.in b/src/externals/pio2/src/flib/piodarray.F90.in
similarity index 97%
rename from externals/pio2/src/flib/piodarray.F90.in
rename to src/externals/pio2/src/flib/piodarray.F90.in
index f39926f96d7..7c80e89df8a 100644
--- a/externals/pio2/src/flib/piodarray.F90.in
+++ b/src/externals/pio2/src/flib/piodarray.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ 'piodarray'
 !>
-!! @file 
+!! @file
 !! @brief Read and write routines for decomposed data.
 !<
 module piodarray
@@ -16,8 +16,8 @@ module piodarray
   private
   public :: pio_read_darray, pio_write_darray, pio_set_buffer_size_limit
 
-  
-!> 
+
+!>
 !! @defgroup PIO_write_darray PIO_write_darray
 !! @brief The overloaded PIO_write_darray writes a distributed array to disk.
 !<
@@ -30,7 +30,7 @@ module piodarray
   end interface
 
 
-!> 
+!>
 !! @defgroup PIO_read_darray PIO_read_darray
 !! @brief The overloaded PIO_read_darray function reads a distributed array from disk.
 !<
@@ -90,9 +90,9 @@ end interface
 contains
 
   subroutine pio_set_buffer_size_limit(limit)
-    integer(PIO_OFFSET_KIND), intent(in) :: limit    
+    integer(PIO_OFFSET_KIND), intent(in) :: limit
     integer(PIO_OFFSET_KIND) :: oldval
-    interface 
+    interface
        integer(C_LONG_LONG) function PIOc_set_buffer_size_limit(limit) &
             bind(C,name="PIOc_set_buffer_size_limit")
          use iso_c_binding
@@ -103,7 +103,7 @@ contains
        call piodie(__PIO_FILE__,__LINE__,' bad value to buffer_size_limit: ',int(limit))
     end if
     oldval = PIOc_set_buffer_size_limit(limit)
-       
+
   end subroutine pio_set_buffer_size_limit
 
 ! TYPE real,int,double
@@ -138,7 +138,7 @@ contains
     carraylen = int(arraylen,C_SIZE_T)
     cptr = C_LOC(array)
 #ifdef TIMING
-    call t_startf("PIO:write_darray_{TYPE}")    
+    call t_startf("PIO:write_darray_{TYPE}")
 #endif
     if(present(fillval)) then
        iostat = PIOc_write_darray(file%fh, varDesc%varid-1, iodesc%ioid, carraylen,cptr, C_LOC(fillval))
@@ -146,7 +146,7 @@ contains
        iostat = PIOc_write_darray(file%fh, varDesc%varid-1, iodesc%ioid, carraylen, cptr, C_NULL_PTR)
     endif
 #ifdef TIMING
-    call t_stopf("PIO:write_darray_{TYPE}")    
+    call t_stopf("PIO:write_darray_{TYPE}")
 #endif
   end subroutine write_darray_1d_cinterface_{TYPE}
 
@@ -183,7 +183,7 @@ contains
     type(C_PTR) :: cptr
     integer :: i
     carraylen = int(arraylen,C_SIZE_T)
-    
+
     cptr = C_LOC(array)
     do i=1,nvars
        varid(i) = vardesc(i)%varid-1
@@ -198,10 +198,10 @@ contains
   end subroutine write_darray_multi_1d_cinterface_{TYPE}
 
 ! TYPE real,int,double
-!> 
+!>
 !! @public
 !! @ingroup PIO_write_darray
-!! @brief Writes a 1D array of type {TYPE}. 
+!! @brief Writes a 1D array of type {TYPE}.
 !! @details
 !! @param File    \ref file_desc_t
 !! @param varDesc \ref var_desc_t
@@ -209,7 +209,7 @@ contains
 !! @param array  : The data to be written
 !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
 !! @param fillval : An optional fill value to fill holes in the data written
-!<  
+!<
   subroutine write_darray_multi_1d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
     ! !DESCRIPTION:
     !  Writes a block of TYPE to a netcdf file.
@@ -238,7 +238,7 @@ contains
     integer :: nvars
 
     nvars = size(vardesc)
-    
+
     call write_darray_multi_1d_cinterface_{TYPE} (file, varDesc, iodesc, nvars, size(array), array, iostat, fillval)
 
   end subroutine write_darray_multi_1d_{TYPE}
@@ -269,17 +269,17 @@ contains
     integer(i4), intent(out) :: iostat
 
     character(len=*), parameter :: subName=modName//'::write_darray_{TYPE}'
-    
+
     call write_darray_1d_cinterface_{TYPE} (file, varDesc, iodesc, size(array), array, iostat, fillval)
 
   end subroutine write_darray_1d_{TYPE}
 
 ! TYPE real,int,double
 ! DIMS 2,3,4,5,6,7
-!> 
+!>
 !! @public
 !! @ingroup PIO_write_darray
-!! @brief Writes a {DIMS}D array of type {TYPE}. 
+!! @brief Writes a {DIMS}D array of type {TYPE}.
 !! @details
 !! @param File @ref file_desc_t
 !! @param varDesc @ref var_desc_t
@@ -287,7 +287,7 @@ contains
 !! @param array  : The data to be written
 !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
 !! @param fillval : An optional fill value to fill holes in the data written
-!<  
+!<
   subroutine write_darray_{DIMS}d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
     ! !INPUT PARAMETERS:
 
@@ -323,7 +323,7 @@ contains
 ! cannot call transfer function with a 0 sized array
     if(size(array)==0) then
        call write_darray_1d_{TYPE} (File, varDesc, iodesc, dumbvar, iostat)
-    else 
+    else
        call write_darray_1d_{TYPE} (File, varDesc, iodesc, transfer(array,transvar), iostat, fillval)
     end if
 #endif
@@ -331,7 +331,7 @@ contains
 
 ! TYPE real,int,double
 ! DIMS 1,2,3,4,5,6,7
-!> 
+!>
 !! @public
 !! @ingroup PIO_read_darray
 !! @brief Read distributed array of type {TYPE} from a netCDF variable of {DIMS} dimension(s).
@@ -386,7 +386,7 @@ contains
     ! !INPUT PARAMETERS:
     integer, intent(in) :: ncid, varid, ioid
     integer(C_SIZE_T), intent(in) :: alen
-    
+
     {VTYPE}, target ::  array(*)                 ! array to be read
 
     integer(i4), intent(out) :: iostat
diff --git a/externals/pio2/src/flib/piolib_mod.F90 b/src/externals/pio2/src/flib/piolib_mod.F90
similarity index 91%
rename from externals/pio2/src/flib/piolib_mod.F90
rename to src/externals/pio2/src/flib/piolib_mod.F90
index 70497c53cf8..e704dab4508 100644
--- a/externals/pio2/src/flib/piolib_mod.F90
+++ b/src/externals/pio2/src/flib/piolib_mod.F90
@@ -12,10 +12,11 @@ module piolib_mod
   use pio_kinds
   !--------------
   use pio_types, only : file_desc_t, iosystem_desc_t, var_desc_t, io_desc_t, &
-	pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
-        pio_noerr, pio_rearr_subset
+        pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
+        pio_noerr, pio_rearr_subset, pio_rearr_opt_t
   !--------------
   use pio_support, only : piodie, debug, debugio, debugasync, checkmpireturn
+  use pio_nf, only : pio_set_log_level
   !
 
 
@@ -50,7 +51,8 @@ module piolib_mod
        PIO_FILE_IS_OPEN, &
        PIO_deletefile, &
        PIO_get_numiotasks, &
-       PIO_iotype_available
+       PIO_iotype_available, &
+       PIO_set_rearr_opts
 
 #ifdef MEMCHK
 !> this is an internal variable for memory leak debugging
@@ -142,7 +144,7 @@ module piolib_mod
 !! @defgroup PIO_initdecomp PIO_initdecomp
 !! @brief PIO_initdecomp is an overload interface the models decomposition to pio.
 !! @details initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4,
-!! and initdecomp_2dof_nf_i4 are all deprecated, but supported for backwards
+!! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards
 !! compatibility.
 !<
   interface PIO_initdecomp
@@ -198,6 +200,7 @@ module piolib_mod
   interface PIO_seterrorhandling
      module procedure seterrorhandlingf
      module procedure seterrorhandlingi
+     module procedure seterrorhandlingg
   end interface
 
 !>
@@ -329,6 +332,7 @@ end subroutine setframe
 !<
   subroutine setdebuglevel(level)
     integer(i4), intent(in) :: level
+    integer :: ierr
     if(level.eq.0) then
        debug=.false.
        debugio=.false.
@@ -357,7 +361,11 @@ subroutine setdebuglevel(level)
        debug=.true.
        debugio=.true.
        debugasync=.true.
-
+    end if
+    ierr = PIO_set_log_level(level)
+    if(ierr /= PIO_NOERR) then
+      ! This is not a fatal error
+      print *, __PIO_FILE__, __LINE__, "Setting log level failed, ierr =",ierr
     end if
   end subroutine setdebuglevel
 
@@ -406,6 +414,35 @@ end function PIOc_Set_IOSystem_Error_Handling
 
   end subroutine seterrorhandlingi
 
+!>
+!! @ingroup PIO_seterrorhandling
+!! @public
+!! @brief set the pio error handling method for the iosystem
+!! @param iosystem : a defined pio system descriptor, see PIO_types
+!! @param method :
+!! @copydoc PIO_error_method
+!<
+  subroutine seterrorhandlingg(global, method, oldmethod)
+    integer, intent(in) :: global
+    integer, intent(in) :: method
+    integer, intent(out), optional :: oldmethod
+
+    interface
+       integer(c_int) function PIOc_Set_IOSystem_Error_Handling(global, method) &
+            bind(C,name="PIOc_Set_IOSystem_Error_Handling")
+         use iso_c_binding
+         integer(c_int), value :: global
+         integer(c_int), value :: method
+       end function PIOc_Set_IOSystem_Error_Handling
+    end interface
+    integer(c_int) ::  loldmethod
+
+    loldmethod = PIOc_Set_IOSystem_Error_Handling(global, method)
+    if(present(oldmethod)) oldmethod = loldmethod
+
+
+  end subroutine seterrorhandlingg
+
 
 !>
 !! @public
@@ -844,8 +881,10 @@ end subroutine PIO_initdecomp_dof_i8
 !! @param iosystem a derived type which can be used in subsequent pio operations (defined in PIO_types).
 !! @param base @em optional argument can be used to offset the first io task - default base is task 1.
 !<
-  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base)
-    use pio_types, only : pio_internal_error
+  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base, rearr_opts)
+    use pio_types, only : pio_internal_error, pio_rearr_opt_t
+    use iso_c_binding
+
     integer(i4), intent(in) :: comp_rank
     integer(i4), intent(in) :: comp_comm
     integer(i4), intent(in) :: num_iotasks
@@ -854,17 +893,21 @@ subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stri
     integer(i4), intent(in) :: rearr
     type (iosystem_desc_t), intent(out)  :: iosystem  ! io descriptor to initalize
     integer(i4), intent(in),optional :: base
+    type (pio_rearr_opt_t), intent(in), optional :: rearr_opts
+
     integer :: lbase
     integer :: ierr
     interface
-       integer(c_int) function PIOc_Init_Intracomm_from_F90(f90_comp_comm, num_iotasks, stride,base,rearr,iosysidp) &
+       integer(c_int) function PIOc_Init_Intracomm_from_F90(f90_comp_comm, num_iotasks, stride,base,rearr,rearr_opts,iosysidp) &
             bind(C,name="PIOc_Init_Intracomm_from_F90")
          use iso_c_binding
+         use pio_types
          integer(C_INT), value :: f90_comp_comm
          integer(C_INT), value :: num_iotasks
          integer(C_INT), value :: stride
          integer(C_INT), value :: base
          integer(C_INT), value :: rearr
+          type(pio_rearr_opt_t) :: rearr_opts
          integer(C_INT) :: iosysidp
        end function PIOc_Init_Intracomm_from_F90
     end interface
@@ -874,7 +917,7 @@ end function PIOc_Init_Intracomm_from_F90
 #endif
     lbase=0
     if(present(base)) lbase=base
-    ierr = PIOc_Init_Intracomm_from_F90(comp_comm,num_iotasks,stride,lbase,rearr,iosystem%iosysid)
+    ierr = PIOc_Init_Intracomm_from_F90(comp_comm,num_iotasks,stride,lbase,rearr,rearr_opts,iosystem%iosysid)
 
     call CheckMPIReturn("Bad Initialization in PIO_Init_Intracomm:  ", ierr,__FILE__,__LINE__)
 #ifdef TIMING
@@ -1212,7 +1255,7 @@ end function pio_iotype_available
 !! @brief  Create a NetCDF or PNetCDF file using PIO.
 !! @details  Input parameters are read on comp task 0 and ignored elsewhere
 !! @param iosystem : A defined pio system descriptor created by a call to @ref PIO_init (see PIO_types)
-!! @param file	:  The returned file descriptor
+!! @param file  :  The returned file descriptor
 !! @param iotype : @copydoc PIO_iotype
 !! @param fname : The name of the file to open
 !! @param amode_in : The NetCDF creation mode flag. the following flags are available:
@@ -1267,7 +1310,7 @@ end function createfile
 !! @brief open an existing file using pio
 !! @details  Input parameters are read on comp task 0 and ignored elsewhere.
 !! @param iosystem : a defined pio system descriptor created by a call to @ref PIO_init (see PIO_types)
-!! @param file	:  the returned file descriptor
+!! @param file  :  the returned file descriptor
 !! @param iotype : @copybrief PIO_iotype
 !! @param fname : the name of the file to open
 !! @param mode : a zero value (or PIO_nowrite) specifies the default
@@ -1482,6 +1525,64 @@ end function PIOc_deletefile
 
   end subroutine pio_deletefile
 
+!>
+!! @public
+!! @ingroup PIO_set_rearr_opts
+!! @brief Set the rerranger options
+!! @details
+!! @param ios : handle to pio iosystem
+!! @param comm_type : @copydoc PIO_rearr_comm_t
+!! @param fcd : @copydoc PIO_rearr_comm_dir
+!! @param enable_hs_c2i : Enable handshake (compute procs to io procs)
+!! @param enable_isend_c2i : Enable isends (compute procs to io procs)
+!! @param max_pend_req_c2i: Maximum pending requests (compute procs to io procs)
+!! @param enable_hs_i2c : Enable handshake (io procs to compute procs)
+!! @param enable_isend_i2c : Enable isends (io procs to compute procs)
+!! @param max_pend_req_i2c: Maximum pending requests (io procs to compute procs)
+!! @copydoc PIO_rearr_comm_fc_options
+!<
+  function pio_set_rearr_opts(ios, comm_type, fcd,&
+                              enable_hs_c2i, enable_isend_c2i,&
+                              max_pend_req_c2i,&
+                              enable_hs_i2c, enable_isend_i2c,&
+                              max_pend_req_i2c) result(ierr)
+
+    type(iosystem_desc_t), intent(inout) :: ios
+    integer, intent(in) :: comm_type, fcd
+    logical, intent(in) :: enable_hs_c2i, enable_hs_i2c
+    logical, intent(in) :: enable_isend_c2i, enable_isend_i2c
+    integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c
+    integer :: ierr
+    interface
+      integer(c_int) function PIOc_set_rearr_opts(iosysid, comm_type, fcd,&
+                                                  enable_hs_c2i, enable_isend_c2i,&
+                                                  max_pend_req_c2i,&
+                                                  enable_hs_i2c, enable_isend_i2c,&
+                                                  max_pend_req_i2c)&
+        bind(C,name="PIOc_set_rearr_opts")
+        use iso_c_binding
+        integer(C_INT), intent(in), value :: iosysid
+        integer(C_INT), intent(in), value :: comm_type
+        integer(C_INT), intent(in), value :: fcd
+        logical(C_BOOL), intent(in), value :: enable_hs_c2i
+        logical(C_BOOL), intent(in), value :: enable_isend_c2i
+        integer(C_INT), intent(in), value :: max_pend_req_c2i
+        logical(C_BOOL), intent(in), value :: enable_hs_i2c
+        logical(C_BOOL), intent(in), value :: enable_isend_i2c
+        integer(C_INT), intent(in), value :: max_pend_req_i2c
+      end function PIOc_set_rearr_opts
+    end interface
+
+    ierr = PIOc_set_rearr_opts(ios%iosysid, comm_type, fcd,&
+                                logical(enable_hs_c2i, kind=c_bool),&
+                                logical(enable_isend_c2i, kind=c_bool),&
+                                max_pend_req_c2i,&
+                                logical(enable_hs_i2c, kind=c_bool),&
+                                logical(enable_isend_i2c, kind=c_bool),&
+                                max_pend_req_i2c)
+
+  end function pio_set_rearr_opts
+
 
 end module piolib_mod
 
diff --git a/externals/pio2/src/flib/pionfatt_mod.F90.in b/src/externals/pio2/src/flib/pionfatt_mod.F90.in
similarity index 95%
rename from externals/pio2/src/flib/pionfatt_mod.F90.in
rename to src/externals/pio2/src/flib/pionfatt_mod.F90.in
index 4066b96c073..daaca6dca81 100644
--- a/externals/pio2/src/flib/pionfatt_mod.F90.in
+++ b/src/externals/pio2/src/flib/pionfatt_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "pionfatt_mod.F90"
 !>
-!! @file 
+!! @file
 !! @brief NetCDF attribute interface to PIO
 !<
 module pionfatt_mod
@@ -37,14 +37,14 @@ module pionfatt_mod
   end interface
 
   !>
-  !!  @public 
+  !!  @public
   !!  @defgroup PIO_put_att PIO_put_att
-  !!  @brief Writes an netcdf attribute to a file 
+  !!  @brief Writes an netcdf attribute to a file
   !<
   !>
-  !! @public 
+  !! @public
   !! @defgroup PIO_get_att PIO_get_att
-  !! @brief Reads  an netcdf attribute from a file 
+  !! @brief Reads  an netcdf attribute from a file
   !<
 
   private :: modName
@@ -121,7 +121,7 @@ module pionfatt_mod
        integer(C_INT), intent(out)   :: op
      end function PIOc_get_att_int
   end interface
-    
+
   interface
      integer(C_INT) function PIOc_get_att_float (ncid, varid, name, op) &
           bind(C,name="PIOc_get_att_float")
@@ -132,7 +132,7 @@ module pionfatt_mod
        real(C_FLOAT), intent(out) :: op
      end function PIOc_get_att_float
   end interface
-    
+
   interface
      integer(C_INT) function PIOc_get_att_double (ncid, varid, name, op) &
           bind(C,name="PIOc_get_att_double")
@@ -143,19 +143,19 @@ module pionfatt_mod
        real(C_DOUBLE), intent(out)   :: op
      end function PIOc_get_att_double
   end interface
-    
+
 
 contains
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to add
-  !! @param var : The value for the netcdf attribute 
+  !! @param var : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_desc_{TYPE} (File, vdesc, name, values) result(ierr)
@@ -208,7 +208,7 @@ contains
     deallocate(cvar)
   end function put_att_id_text
 
-  integer function put_att_1d_id_text (ncid, varid, name, value) result(ierr)  
+  integer function put_att_1d_id_text (ncid, varid, name, value) result(ierr)
     use iso_c_binding
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
@@ -220,8 +220,8 @@ contains
     slen = len(value(1))
     alen = size(value)
     allocate(nvalue(slen*alen))
-    
-    do i=1,alen 
+
+    do i=1,alen
        j= len_trim(value(i))
        do k=1,j
           nvalue(k+(i-1)*slen) = value(i)(k:k)
@@ -244,9 +244,9 @@ contains
     character(len=*), intent(in) :: values(arrlen)
     integer :: vallen
 
-    
+
     ierr = PIOc_put_att_text (ncid,varid-1,trim(name)//C_NULL_CHAR, int(arrlen,C_SIZE_T),values(1))
-        
+
 
   end function put_att_1d_id_text_internal
 
@@ -267,14 +267,14 @@ contains
   !pl The next line is needed by genf90.pl, do not remove it.
   ! TYPE real,double,int
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to add
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_1d_id_{TYPE} (ncid, varid, name, values) result(ierr)
@@ -305,14 +305,14 @@ contains
 
   ! TYPE real,int,double
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to add
-  !! @param var : The value for the netcdf attribute 
+  !! @param var : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_1d_desc_{TYPE} (File,varDesc,name,values) result(ierr)
@@ -339,14 +339,14 @@ contains
 
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function get_att_desc_{TYPE} (File,varDesc,name,values) result(ierr)
@@ -361,14 +361,14 @@ contains
   end function get_att_desc_{TYPE}
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   ! TYPE int,real,double
@@ -386,14 +386,14 @@ contains
   end function get_att_desc_1d_{TYPE}
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   ! TYPE int,real,double
@@ -410,7 +410,7 @@ contains
   end function get_att_id_{TYPE}
 
 
-  
+
   integer function get_att_{TYPE} (File,varid,name,values) result(ierr)
     type (File_desc_t), intent(in) , target :: File
     integer(i4), intent(in)        :: varid
@@ -425,14 +425,14 @@ contains
 
   ! TYPE real,int,double
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function get_att_1d_{TYPE} (File,varid,name,values) result(ierr)
@@ -460,4 +460,3 @@ contains
 
 
 end module pionfatt_mod
-
diff --git a/externals/pio2/src/flib/pionfget_mod.F90.in b/src/externals/pio2/src/flib/pionfget_mod.F90.in
similarity index 98%
rename from externals/pio2/src/flib/pionfget_mod.F90.in
rename to src/externals/pio2/src/flib/pionfget_mod.F90.in
index 3cbca7e6bed..ebd335dde2e 100644
--- a/externals/pio2/src/flib/pionfget_mod.F90.in
+++ b/src/externals/pio2/src/flib/pionfget_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "pionfget_mod.F90"
 !>
-!! @file 
+!! @file
 !! @brief Read Routines for non-decomposed NetCDF data.
 !<
 module pionfget_mod
@@ -17,8 +17,8 @@ module pionfget_mod
 !! @defgroup PIO_get_var PIO_get_var
 !! @brief Reads non-decomposed data from a NetCDF file
 !! @details The get_var interface is provided as a simplified interface to
-!!  read variables from a NetCDF format file.   The variable is read on the 
-!!  root IO task and broadcast in its entirety to all tasks.  
+!!  read variables from a NetCDF format file.   The variable is read on the
+!!  root IO task and broadcast in its entirety to all tasks.
 !<
   public :: get_var
   interface get_var
@@ -284,7 +284,7 @@ CONTAINS
 !!   the variable's dimensions. Hence, if the variable is a record
 !!   variable, the first element of count corresponds to a count of the
 !!   number of records to read.
-!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything. 
+!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything.
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @ref error_return
 !<
@@ -332,7 +332,7 @@ CONTAINS
 !!   the variable's dimensions. Hence, if the variable is a record
 !!   variable, the first element of count corresponds to a count of the
 !!   number of records to read.
-!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything. 
+!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything.
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @ref error_return
 !<
@@ -379,11 +379,11 @@ CONTAINS
     integer, intent(in) :: varid
     {VTYPE}, intent(out) :: ival
     {VTYPE}  :: aival(1)
-    
+
 
     ierr = PIOc_get_var_{NCTYPE} (File%fh, varid-1, aival)
     ival = aival(1)
-       
+
   end function Get_var_0d_{TYPE}
 
 ! DIMS 1,2,3,4,5
@@ -395,7 +395,7 @@ CONTAINS
     ierr = get_var_text_internal(File%fh, varid, size(ival), ival)
 
   end function get_var_{DIMS}d_text
- 
+
   integer function get_var_text_internal (ncid,varid, nstrs, ival) result(ierr)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
diff --git a/externals/pio2/src/flib/pionfput_mod.F90.in b/src/externals/pio2/src/flib/pionfput_mod.F90.in
similarity index 98%
rename from externals/pio2/src/flib/pionfput_mod.F90.in
rename to src/externals/pio2/src/flib/pionfput_mod.F90.in
index fdc3e624197..d4b54220a94 100644
--- a/externals/pio2/src/flib/pionfput_mod.F90.in
+++ b/src/externals/pio2/src/flib/pionfput_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "pionfput_mod.F90"
 !>
-!! @file 
+!! @file
 !! @brief Write routines for non-decomposed NetCDF data.
 !<
 module pionfput_mod
@@ -18,17 +18,17 @@ module pionfput_mod
 !! @defgroup PIO_put_var PIO_put_var
 !! @brief Writes data to a netCDF file.
 !! @details The put_var interface is provided as a simplified interface to
-!!  write variables to a netcdf format file.   
-!! @warning Although this is a collective call the variable is written from the 
+!!  write variables to a netcdf format file.
+!! @warning Although this is a collective call the variable is written from the
 !!  root IO task, no consistancy check is made with data passed on other tasks.
-!! 
+!!
 !<
   public :: put_var
   interface put_var
      !  DIMS 0,1,2,3,4,5
      module procedure put_var_{DIMS}d_{TYPE}, put_var_vdesc_{DIMS}d_{TYPE}
      !  DIMS 1,2,3,4,5
-     module procedure put_vara_{DIMS}d_{TYPE} 
+     module procedure put_vara_{DIMS}d_{TYPE}
      !  DIMS 1,2,3,4,5
      module procedure put_vara_vdesc_{DIMS}d_{TYPE}
      module procedure put_var1_{TYPE}, put_var1_vdesc_{TYPE}
@@ -99,7 +99,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param varid : The netcdf variable identifier
-!! @param index : 
+!! @param index :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -111,7 +111,7 @@ contains
     integer :: i
     integer, allocatable :: count(:)
     integer :: ndims
-    
+
     ndims = size(index)
     allocate(count(ndims))
     count = 1
@@ -134,7 +134,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param varid : The netcdf variable identifier
-!! @param index : 
+!! @param index :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -157,18 +157,18 @@ contains
 
 #ifdef TIMING
     call t_startf("PIO:put_var1_{TYPE}")
-#endif 
+#endif
     clen = size(index)
     allocate(cindex(clen))
     do i=1,clen
        cindex(i) = index(clen-i+1)-1
     enddo
-    
+
     ierr = PIOc_put_var1_{NCTYPE} (file%fh, varid-1, cindex, ival)
     deallocate(cindex)
 #ifdef TIMING
     call t_stopf("PIO:put_var1_{TYPE}")
-#endif 
+#endif
   end function put_var1_{TYPE}
 
 !>
@@ -178,7 +178,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param start : 
+!! @param start :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -220,7 +220,7 @@ contains
     do i=1,len_trim(ival)
        cval(i) = ival(i:i)
     end do
-    
+
     ierr = PIOc_put_var_text(file%fh, varid-1,  cval)
 
     deallocate(cval)
@@ -276,7 +276,7 @@ contains
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
     {VTYPE}, intent(in) :: ival(*)
-    
+
     interface
        integer(C_INT) function PIOc_put_var_{NCTYPE}(ncid, varid, op) &
             bind(C,name="PIOc_put_var_{NCTYPE}")
@@ -338,13 +338,13 @@ contains
     ierr=PIO_NOERR
 #ifdef TIMING
     call t_startf("PIO:put_var_0d_{TYPE}")
-#endif 
+#endif
 
     ierr = put_var_internal_{TYPE} (File%fh, varid, (/ival/))
 
 #ifdef TIMING
     call t_stopf("PIO:put_var_0d_{TYPE}")
-#endif 
+#endif
   end function put_var_0d_{TYPE}
 
 
@@ -419,7 +419,7 @@ contains
     integer(C_SIZE_T), allocatable :: cstart(:), ccount(:)
     integer :: i
     integer :: ndims
-   
+
     do i=1,size(count)
        if(count(i)<=0) then
           ndims=i-1
@@ -528,7 +528,7 @@ contains
     enddo
 #endif
 
-    
+
   end subroutine Fstring2Cstring_{DIMS}d
 
 
@@ -554,13 +554,13 @@ contains
     {VTYPE}, intent(in) :: ival{DIMSTR}
 #ifdef TIMING
     call t_startf("PIO:put_vara_{DIMS}d_{TYPE}")
-#endif 
+#endif
 
     ierr = put_vara_internal_{TYPE} (File%fh, varid, start, count, ival)
 
 #ifdef TIMING
     call t_stopf("PIO:put_vara_{DIMS}d_{TYPE}")
-#endif 
+#endif
   end function put_vara_{DIMS}d_{TYPE}
 
 ! DIMS 1,2,3,4,5
@@ -571,8 +571,8 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param start : 
-!! @param count : 
+!! @param start :
+!! @param count :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
diff --git a/src/externals/pio2/src/gptl/CMakeLists.txt b/src/externals/pio2/src/gptl/CMakeLists.txt
new file mode 100644
index 00000000000..5f73687f23d
--- /dev/null
+++ b/src/externals/pio2/src/gptl/CMakeLists.txt
@@ -0,0 +1,156 @@
+cmake_minimum_required (VERSION 2.8.12)
+project (GPTL C Fortran)
+include (CheckFunctionExists)
+include (FortranCInterface)
+FortranCInterface_HEADER(cmake_fortran_c_interface.h
+  MACRO_NAMESPACE "FCI_")
+include (LibCheck)
+
+#==============================================================================
+#  DEFINE THE TARGET
+#==============================================================================
+
+set (GPTL_C_SRCS GPTLget_memusage.c
+  GPTLprint_memusage.c
+  GPTLutil.c
+  f_wrappers.c
+  gptl.c
+  gptl_papi.c
+  threadutil.c)
+
+set (GPTL_Fortran_SRCS perf_mod.F90
+  perf_utils.F90)
+
+set (GPTL_Fortran_MODS ${CMAKE_CURRENT_BINARY_DIR}/perf_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/perf_utils.mod)
+
+add_library (gptl ${GPTL_Fortran_SRCS} ${GPTL_C_SRCS})
+
+target_include_directories (gptl
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
+  PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+target_compile_definitions (gptl
+  PUBLIC INCLUDE_CMAKE_FCI)
+
+if (CMAKE_SYSTEM_NAME MATCHES "AIX")
+  target_compile_definitions (gptl
+    PUBLIC _AIX)
+else ()
+  target_compile_definitions (gptl
+    PUBLIC ${CMAKE_SYSTEM_DIRECTIVE})
+endif ()
+
+target_compile_definitions (gptl
+  PUBLIC ${CMAKE_Fortran_COMPILER_DIRECTIVE})
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
+endif ()
+
+#==============================================================================
+#  DEFINE THE INSTALL
+#==============================================================================
+
+# Library
+install (TARGETS gptl DESTINATION lib)
+
+# Header/Include File
+install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/gptl.h DESTINATION include)
+
+# Fortran Modules
+install (FILES ${GPTL_Fortran_MODS} DESTINATION include)
+
+#==============================================================================
+#  DEFINE THE DEPENDENCIES
+#==============================================================================
+
+#===== PAPI =====
+if (ENABLE_PAPI)
+  find_package (PAPI)
+  if (PAPI_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC HAVE_PAPI)
+    target_include_directories (gptl
+      PUBLIC ${PAPI_INCLUDE_DIRECTORIES})
+    target_link_libraries (gptl
+      PUBLIC ${PAPI_LIBRARIES})
+  endif ()
+endif ()
+
+#===== LIBRT =====
+if (ENABLE_LIBRT)
+  find_package (LIBRT)
+  if (LIBRT_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC HAVE_LIBRT)
+    target_include_directories (gptl
+      PUBLIC ${LIBRT_INCLUDE_DIRECTORIES})
+    target_link_libraries (gptl
+      PUBLIC ${LIBRT_LIBRARIES})
+  endif ()
+endif ()
+
+#===== MPI =====
+if (PIO_USE_MPISERIAL)
+  find_package (MPISERIAL COMPONENTS C Fortran REQUIRED)
+  if (MPISERIAL_C_FOUND AND MPISERIAL_Fortran_FOUND)
+    target_compile_definitions (gptl
+      PRIVATE HAVE_MPI)
+    target_include_directories (gptl
+      PUBLIC ${MPISERIAL_C_INCLUDE_DIRS}
+      PUBLIC ${MPISERIAL_Fortran_INCLUDE_DIRS})
+    target_link_libraries (gptl
+      PUBLIC ${MPISERIAL_C_LIBRARIES}
+      PUBLIC ${MPISERIAL_Fortran_LIBRARIES})
+
+    set (MPI_C_LIBRARIES ${MPISERIAL_C_LIBRARIES})
+    set (MPI_Fortran_INCLUDE_PATH ${MPISERIAL_Fortran_INCLUDE_DIRS})
+  endif ()
+else ()
+  find_package (MPI REQUIRED)
+  if (MPI_C_FOUND AND MPI_Fortran_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC HAVE_MPI)
+  endif ()
+endif ()
+
+# Check MPI library for Comm_f2c function
+set (CMAKE_REQUIRED_LIBRARIES ${MPI_C_LIBRARIES})
+check_function_exists (MPI_Comm_f2c MPI_HAS_COMM_F2C)
+if (NOT MPI_HAS_COMM_F2C)
+  target_compile_definitions(gptl
+    PRIVATE NO_COMM_F2C)
+endif ()
+
+# Check for MPI Fortran module
+find_path(MPIMOD_PATH
+  NAMES mpi.mod MPI.mod
+  PATHS ${MPI_Fortran_INCLUDE_PATH})
+check_macro (MPI_HAS_Fortran_MOD
+  NAME TryMPIMod.f90
+  HINTS ${CMAKE_MODULE_PATH}
+  DEFINITIONS -I${MPIMOD_PATH}
+  COMMENT "whether MPI Fortran module is supported")
+if (${MPI_HAS_Fortran_MOD})
+  message (STATUS "MPI Fortran module verified and enabled.")
+else ()
+  message (STATUS "MPI Fortran module failed verification and therefore disabled.")
+  target_compile_definitions (gptl
+    PUBLIC NO_MPIMOD)
+endif ()
+
+#===== GetTimeOfDay =====
+if (NOT DEFINED SYSTEM_HAS_GETTIMEOFDAY)
+  get_target_property (GPTL_LINK_LIBRARIES gptl LINK_LIBRARIES)
+  set (CMAKE_REQUIRED_LIBRARIES ${GPTL_LINK_LIBRARIES})
+  check_function_exists (gettimeofday GETTIMEOFDAY)
+  if (NOT GETTIMEOFDAY)
+    target_compile_definitions(gptl
+      PUBLIC NO_GETTIMEOFDAY)
+  endif ()
+  set (SYSTEM_HAS_GETTIMEOFDAY ${GETTIMEOFDAY}
+    CACHE INTERNAL "Whether the gettimeofday function could be found")
+endif ()
diff --git a/src/externals/pio2/src/gptl/COPYING b/src/externals/pio2/src/gptl/COPYING
new file mode 100644
index 00000000000..324ce86b24c
--- /dev/null
+++ b/src/externals/pio2/src/gptl/COPYING
@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software for any noncommercial purposes without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following
+conditions: The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.  Any
+commercial use (including sale) of the software, and derivative development
+towards commercial use, requires written permission of the copyright
+holder. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
+OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/externals/pio2/src/gptl/ChangeLog b/src/externals/pio2/src/gptl/ChangeLog
similarity index 100%
rename from externals/pio2/src/gptl/ChangeLog
rename to src/externals/pio2/src/gptl/ChangeLog
diff --git a/externals/pio2/src/gptl/GPTLget_memusage.c b/src/externals/pio2/src/gptl/GPTLget_memusage.c
similarity index 100%
rename from externals/pio2/src/gptl/GPTLget_memusage.c
rename to src/externals/pio2/src/gptl/GPTLget_memusage.c
diff --git a/externals/pio2/src/gptl/GPTLprint_memusage.c b/src/externals/pio2/src/gptl/GPTLprint_memusage.c
similarity index 100%
rename from externals/pio2/src/gptl/GPTLprint_memusage.c
rename to src/externals/pio2/src/gptl/GPTLprint_memusage.c
diff --git a/externals/pio2/src/gptl/GPTLutil.c b/src/externals/pio2/src/gptl/GPTLutil.c
similarity index 100%
rename from externals/pio2/src/gptl/GPTLutil.c
rename to src/externals/pio2/src/gptl/GPTLutil.c
diff --git a/externals/pio2/src/gptl/README b/src/externals/pio2/src/gptl/README
similarity index 100%
rename from externals/pio2/src/gptl/README
rename to src/externals/pio2/src/gptl/README
diff --git a/externals/pio2/src/gptl/f_wrappers.c b/src/externals/pio2/src/gptl/f_wrappers.c
similarity index 100%
rename from externals/pio2/src/gptl/f_wrappers.c
rename to src/externals/pio2/src/gptl/f_wrappers.c
diff --git a/externals/pio2/src/gptl/gptl.c b/src/externals/pio2/src/gptl/gptl.c
similarity index 99%
rename from externals/pio2/src/gptl/gptl.c
rename to src/externals/pio2/src/gptl/gptl.c
index 6346bf1b999..d06d33f0883 100644
--- a/externals/pio2/src/gptl/gptl.c
+++ b/src/externals/pio2/src/gptl/gptl.c
@@ -195,8 +195,8 @@ static int update_ll_hash (Timer *, const int, const unsigned int);
 static inline int update_ptr (Timer *, const int);
 static int construct_tree (Timer *, Method);
 
-static int cmp (const char **, const char **);
-static int ncmp (const char **, const char **);
+static int cmp (const void *, const void *);
+static int ncmp (const void *, const void *);
 static int get_index ( const char *, const char *);
 
 typedef struct {
@@ -2933,9 +2933,9 @@ int get_index( const char * list,
 ** cmp: returns value from strcmp. for use with qsort
 */
 
-static int cmp(const char **x, const char **y)
+static int cmp(const void *x, const void *y)
 {
-  return strcmp(*x, *y);
+  return strcmp(*(char**)x, *(char**)y);
 }
 
 
@@ -2943,15 +2943,17 @@ static int cmp(const char **x, const char **y)
 ** ncmp: compares values of memory adresses pointed to by a pointer. for use with qsort
 */
 
-static int ncmp( const char **x, const char **y )
+static int ncmp( const void *x, const void *y )
 {
   static const char *thisfunc = "GPTLsetoption";
+  const char **ix = (const char **)x;
+  const char **iy = (const char **)y;
 
-  if( *x > *y )
+  if( *ix > *iy )
     return 1;
-  if( *x < *y )
+  if( *ix < *iy )
     return -1;
-  if( *x == *y )
+  if( *ix == *iy )
     GPTLerror("%s: shared memory address between timers\n", thisfunc);
 }
 
diff --git a/externals/pio2/src/gptl/gptl.h b/src/externals/pio2/src/gptl/gptl.h
similarity index 100%
rename from externals/pio2/src/gptl/gptl.h
rename to src/externals/pio2/src/gptl/gptl.h
diff --git a/externals/pio2/src/gptl/gptl.inc b/src/externals/pio2/src/gptl/gptl.inc
similarity index 100%
rename from externals/pio2/src/gptl/gptl.inc
rename to src/externals/pio2/src/gptl/gptl.inc
diff --git a/externals/pio2/src/gptl/gptl_papi.c b/src/externals/pio2/src/gptl/gptl_papi.c
similarity index 100%
rename from externals/pio2/src/gptl/gptl_papi.c
rename to src/externals/pio2/src/gptl/gptl_papi.c
diff --git a/externals/pio2/src/gptl/perf_mod.F90 b/src/externals/pio2/src/gptl/perf_mod.F90
similarity index 99%
rename from externals/pio2/src/gptl/perf_mod.F90
rename to src/externals/pio2/src/gptl/perf_mod.F90
index 2e38c491c67..e62059de98e 100644
--- a/externals/pio2/src/gptl/perf_mod.F90
+++ b/src/externals/pio2/src/gptl/perf_mod.F90
@@ -30,7 +30,7 @@ module perf_mod
 !-----------------------------------------------------------------------
    implicit none
    private                   ! Make the default access private
-   save
+
 
 !-----------------------------------------------------------------------
 ! Public interfaces ----------------------------------------------------
diff --git a/externals/pio2/src/gptl/perf_utils.F90 b/src/externals/pio2/src/gptl/perf_utils.F90
similarity index 100%
rename from externals/pio2/src/gptl/perf_utils.F90
rename to src/externals/pio2/src/gptl/perf_utils.F90
diff --git a/externals/pio2/src/gptl/private.h b/src/externals/pio2/src/gptl/private.h
similarity index 100%
rename from externals/pio2/src/gptl/private.h
rename to src/externals/pio2/src/gptl/private.h
diff --git a/externals/pio2/src/gptl/threadutil.c b/src/externals/pio2/src/gptl/threadutil.c
similarity index 100%
rename from externals/pio2/src/gptl/threadutil.c
rename to src/externals/pio2/src/gptl/threadutil.c
diff --git a/externals/pio2/tests/CMakeLists.txt b/src/externals/pio2/tests/CMakeLists.txt
similarity index 58%
rename from externals/pio2/tests/CMakeLists.txt
rename to src/externals/pio2/tests/CMakeLists.txt
index 3a210e3c12f..115c732b51c 100644
--- a/externals/pio2/tests/CMakeLists.txt
+++ b/src/externals/pio2/tests/CMakeLists.txt
@@ -5,21 +5,24 @@ project (PIOTests C Fortran)
 #  TESTING TARGET
 #==============================================================================
 
-# Custom "piotests" target (builds the test executables)
-add_custom_target (tests)
+# # Custom "piotests" target (builds the test executables)
+# add_custom_target (tests)
 
-# Custom "check" target that depends upon "tests"
-add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND})
-add_dependencies (check tests)
+# # Custom "check" target that depends upon "tests"
+# add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND})
+# add_dependencies (check tests)
 
 #==============================================================================
 #  INCLUDE SOURCE DIRECTORIES
 #==============================================================================
+add_subdirectory (cunit)
 
-add_subdirectory (unit)
-add_subdirectory (general)
-if (PIO_ENABLE_TIMING)
+if (PIO_ENABLE_FORTRAN)
+  add_subdirectory (unit)
+  add_subdirectory (general)
+  if (PIO_ENABLE_TIMING)
     add_subdirectory (performance)
-else ()
+  else ()
     message (STATUS "Cannot build performance test without gptl timing library")
-endif ()
+  endif ()
+endif()
diff --git a/src/externals/pio2/tests/cunit/CMakeLists.txt b/src/externals/pio2/tests/cunit/CMakeLists.txt
new file mode 100644
index 00000000000..1ef8be7afc3
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/CMakeLists.txt
@@ -0,0 +1,188 @@
+include (LibMPI)
+
+include_directories("${CMAKE_SOURCE_DIR}/tests/cunit")
+
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+endif()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
+
+#==============================================================================
+#  PREPARE FOR TESTING
+#==============================================================================
+
+# Don't run these tests if we are using MPI SERIAL.
+if (NOT PIO_USE_MPISERIAL)
+  add_executable (test_intercomm2 EXCLUDE_FROM_ALL test_intercomm2.c test_common.c)
+  target_link_libraries (test_intercomm2 pioc)
+  add_dependencies (tests test_intercomm2)
+  add_executable (test_async_simple EXCLUDE_FROM_ALL test_async_simple.c test_common.c)
+  target_link_libraries (test_async_simple pioc)
+  add_dependencies (tests test_async_simple)
+  add_executable (test_async_3proc EXCLUDE_FROM_ALL test_async_3proc.c test_common.c)
+  target_link_libraries (test_async_3proc pioc)
+  add_dependencies (tests test_async_3proc)
+  add_executable (test_async_4proc EXCLUDE_FROM_ALL test_async_4proc.c test_common.c)
+  target_link_libraries (test_async_4proc pioc)
+  add_dependencies (tests test_async_4proc)
+  add_executable (test_iosystem2_simple EXCLUDE_FROM_ALL test_iosystem2_simple.c test_common.c)
+  target_link_libraries (test_iosystem2_simple pioc)
+  add_dependencies (tests test_iosystem2_simple)
+  add_executable (test_iosystem2_simple2 EXCLUDE_FROM_ALL test_iosystem2_simple2.c test_common.c)
+  target_link_libraries (test_iosystem2_simple2 pioc)
+  add_dependencies (tests test_iosystem2_simple2)
+  add_executable (test_iosystem2 EXCLUDE_FROM_ALL test_iosystem2.c test_common.c)
+  target_link_libraries (test_iosystem2 pioc)
+  add_dependencies (tests test_iosystem2)
+  add_executable (test_iosystem3_simple EXCLUDE_FROM_ALL test_iosystem3_simple.c test_common.c)
+  target_link_libraries (test_iosystem3_simple pioc)
+  add_dependencies (tests test_iosystem3_simple)
+  add_executable (test_iosystem3_simple2 EXCLUDE_FROM_ALL test_iosystem3_simple2.c test_common.c)
+  target_link_libraries (test_iosystem3_simple2 pioc)
+  add_dependencies (tests test_iosystem3_simple2)
+  add_executable (test_iosystem3 EXCLUDE_FROM_ALL test_iosystem3.c test_common.c)
+  target_link_libraries (test_iosystem3 pioc)
+  add_dependencies (tests test_iosystem3)
+  add_executable (test_pioc EXCLUDE_FROM_ALL test_pioc.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc pioc)
+  add_executable (test_pioc_unlim EXCLUDE_FROM_ALL test_pioc_unlim.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc_unlim pioc)
+  add_executable (test_pioc_putget EXCLUDE_FROM_ALL test_pioc_putget.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc_putget pioc)
+  add_executable (test_pioc_fill EXCLUDE_FROM_ALL test_pioc_fill.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc_fill pioc)
+  add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c test_common.c)
+  target_link_libraries (test_darray pioc)
+  add_executable (test_darray_multivar EXCLUDE_FROM_ALL test_darray_multivar.c test_common.c)
+  target_link_libraries (test_darray_multivar pioc)
+  add_executable (test_darray_1d EXCLUDE_FROM_ALL test_darray_1d.c test_common.c)
+  target_link_libraries (test_darray_1d pioc)  
+  add_executable (test_darray_3d EXCLUDE_FROM_ALL test_darray_3d.c test_common.c)
+  target_link_libraries (test_darray_3d pioc)  
+  add_executable (test_decomps EXCLUDE_FROM_ALL test_decomps.c test_common.c)
+  target_link_libraries (test_decomps pioc)
+endif ()
+add_executable (test_spmd EXCLUDE_FROM_ALL test_spmd.c test_common.c)
+target_link_libraries (test_spmd pioc)
+add_dependencies (tests test_spmd)
+add_dependencies (tests test_pioc)
+add_dependencies (tests test_pioc_unlim)
+add_dependencies (tests test_pioc_putget)
+add_dependencies (tests test_pioc_fill)
+add_dependencies (tests test_darray)
+add_dependencies (tests test_darray_multivar)
+add_dependencies (tests test_darray_1d)
+add_dependencies (tests test_darray_3d)
+add_dependencies (tests test_decomps)
+
+# Test Timeout in seconds.
+if (PIO_VALGRIND_CHECK)
+  set (DEFAULT_TEST_TIMEOUT 120)
+else ()
+  set (DEFAULT_TEST_TIMEOUT 60)
+endif ()
+
+# All tests need a certain number of tasks, but they should be able to
+# run successfully with more than they need. Test this by providing an
+# extra processor for each C test.
+set (AT_LEAST_TWO_TASKS 3)
+set (AT_LEAST_THREE_TASKS 4)
+set (AT_LEAST_FOUR_TASKS 5)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME test_pioc
+    COMMAND test_pioc)
+else ()
+  add_mpi_test(test_spmd
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_spmd
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_intercomm2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_intercomm2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_async_simple
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_async_simple
+    NUMPROCS ${AT_LEAST_TWO_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_async_3proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_async_3proc
+    NUMPROCS ${AT_LEAST_THREE_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_async_4proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_async_4proc
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem2_simple
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem2_simple
+    NUMPROCS ${AT_LEAST_TWO_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem2_simple2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem2_simple2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem3_simple
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem3_simple
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem3_simple2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem3_simple2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem3
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem3
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc_unlim
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_unlim
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc_putget
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_putget
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc_fill
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_fill
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_multivar
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_1d
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_1d
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_3d
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_3d
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_decomps
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomps
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
diff --git a/src/externals/pio2/tests/cunit/pio_tests.h b/src/externals/pio2/tests/cunit/pio_tests.h
new file mode 100644
index 00000000000..86ac35af547
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/pio_tests.h
@@ -0,0 +1,100 @@
+/**
+ * @file
+ * Include file for tests for the Parallel IO library.
+ * @author Ed Hartnett
+ * @date 9/13/2016
+ */
+
+#ifndef _PIO_TESTS_H
+#define _PIO_TESTS_H
+
+#include  /* Include this for the sleep function. */
+#include 
+
+/* Timing header may need to be included. */
+#ifdef TIMING
+#include 
+#endif
+
+/** The number of possible output netCDF output flavors available to
+ * the ParallelIO library. */
+#define NUM_FLAVORS 4
+
+/** Number of netCDF types. */
+#define NUM_NETCDF_TYPES 12
+
+/* Number of NetCDF classic types. */
+#define NUM_CLASSIC_TYPES 6
+
+/* Number of NetCDF-4 types. */
+#define NUM_NETCDF4_TYPES 12
+
+/* Number of sample files constructed for these tests. */
+#define NUM_SAMPLES 3
+
+/** Error code for when things go wrong. */
+#define ERR_CHECK 1109
+#define ERR_INIT 1110
+#define ERR_AWFUL 1111
+#define ERR_WRONG 1112
+#define ERR_GPTL 1113
+#define ERR_MPI 1114
+
+/** The meaning of life, the universe, and everything. */
+#define TEST_VAL_42 42
+
+/** Handle MPI errors. This should only be used with MPI library
+ * function calls. */
+#define MPIERR(e) do {                                                  \
+        MPI_Error_string(e, err_buffer, &resultlen);                    \
+        fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
+        MPI_Finalize();                                                 \
+        return ERR_AWFUL;                                               \
+    } while (0)
+
+/** Handle non-MPI errors by finalizing the MPI library and exiting
+ * with an exit code. */
+#define ERR(e) do {                                                     \
+        fprintf(stderr, "%d Error %d in %s, line %d\n", my_rank, e, __FILE__, __LINE__); \
+        MPI_Finalize();                                                 \
+        return e;                                                       \
+    } while (0)
+
+/** Global err buffer for MPI. When there is an MPI error, this buffer
+ * is used to store the error message that is associated with the MPI
+ * error. */
+char err_buffer[MPI_MAX_ERROR_STRING];
+
+/** This is the length of the most recent MPI error message, stored
+ * int the global error string. */
+int resultlen;
+
+/* Function prototypes. */
+int pio_test_init(int argc, char **argv, int *my_rank, int *ntasks, int target_ntasks, MPI_Comm *test_comm);
+int pio_test_init2(int argc, char **argv, int *my_rank, int *ntasks, int min_ntasks,
+                   int max_ntasks, int log_level, MPI_Comm *test_comm);
+int create_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid);
+int create_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int create_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int get_iotypes(int *num_flavors, int *flavors);
+int get_iotype_name(int iotype, char *name);
+int pio_test_finalize(MPI_Comm *test_comm);
+int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm,
+                int component_count, int num_io_procs, int target_ntasks, char *test_name);
+int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm, int target_ntasks,
+                   int x_dim_len, int y_dim_len);
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async);
+int run_test_main(int argc, char **argv, int min_ntasks, int max_ntasks,
+                  int log_level, char *test_name, int *dim_len, int component_count,
+                  int num_io_procs);
+
+/* Create a 2D decomposition used in some tests. */
+int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2d, int *ioid,
+                            int pio_type);
+#endif /* _PIO_TESTS_H */
diff --git a/src/externals/pio2/tests/cunit/test_async_2comp.c b/src/externals/pio2/tests/cunit/test_async_2comp.c
new file mode 100644
index 00000000000..3c79cb6101e
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_2comp.c
@@ -0,0 +1,132 @@
+/**
+ * @file Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ * @author Ed Hartnett
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_intercomm3
    
+ *
+ */
+#include 
+#include 
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 2
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 2
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_intercomm3"
+
+/** Run Tests for Init_Intercomm
+ *
+ */
+int
+main(int argc, char **argv)
+{
+    int verbose = 1;
+
+    /* Zero-based rank of processor. */
+    int my_rank;
+
+    /* Number of processors involved in current execution. */
+    int ntasks;
+
+    /* Different output flavors. */
+    int flavor[NUM_FLAVORS];
+
+    int num_flavors;
+
+    /* The ID for the parallel I/O system. */
+    int iosysid[COMPONENT_COUNT];
+
+    /* The ncid of the netCDF file. */
+    int ncid;
+
+    /* The ID of the netCDF varable. */
+    int varid;
+
+    /* Return code. */
+    int ret;
+
+    /* Index for loops. */
+    int fmt, d, d1, i;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS)))
+        ERR(ERR_INIT);
+
+    /* Figure out iotypes. */
+    if ((ret = get_iotypes(&num_flavors, flavor)))
+        ERR(ret);
+
+    /* How many processors will be used for our IO and 2 computation components. */
+    int num_procs[COMPONENT_COUNT + 1] = {2, 1, 1};
+
+    /* Is the current process a computation task? */
+    int comp_task = my_rank < 2 ? 0 : 1;
+
+    /* Index of computation task in iosysid array. Varies by rank and
+     * does not apply to IO component processes. */
+    int my_comp_idx = comp_task ? my_rank - 2 : -1;
+
+    /* Initialize the IO system. */
+    if ((ret = PIOc_Init_Async(MPI_COMM_WORLD, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                               num_procs, NULL, iosysid)))
+        ERR(ERR_AWFUL);
+
+    /* All the netCDF calls are only executed on the computation
+     * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+     * and when the do, they should go straight to finalize. */
+    if (comp_task)
+    {
+        for (int flv = 0; flv < num_flavors; flv++)
+        {
+            char filename[NC_MAX_NAME + 1];
+
+            /* Create a filename. */
+            int sample = 1;
+            sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, flavor_name(flv), sample, my_comp_idx);
+
+            /* Create sample file 1. */
+            printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+            if ((ret = create_nc_sample_1(iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                ERR(ret);
+
+            /* Check the file for correctness. */
+            if ((ret = check_nc_sample_1(iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                ERR(ret);
+
+        } /* next netcdf format flavor */
+
+        /* If I don't sleep here for a second, there are problems. */
+        sleep(2);
+
+        /* Finalize the IO system. Only call this from the computation tasks. */
+        if (verbose)
+            printf("%d test_intercomm3 Freeing PIO resources\n", my_rank);
+        for (int c = 0; c < COMPONENT_COUNT; c++)
+        {
+            if ((ret = PIOc_finalize(iosysid[c])))
+                ERR(ret);
+            printf("%d test_intercomm3 PIOc_finalize completed for iosysid = %d\n", my_rank, iosysid[c]);
+        }
+    } /* endif comp_task */
+
+    /* Wait for everyone to catch up. */
+    printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME);
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_async_3proc.c b/src/externals/pio2/tests/cunit/test_async_3proc.c
new file mode 100644
index 00000000000..020ed78172c
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_3proc.c
@@ -0,0 +1,128 @@
+/*
+ * Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ *
+ * This very simple test runs on 32 ranks. Eight are used for IO, the
+ * other 24 for computation. The netCDF sample files are created and
+ * checked.
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_async_8io_24comp
    
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 3
+
+/* The name of this test. */
+#define TEST_NAME "test_async_3proc"
+
+/* Number of different combonations of IO and computation processor
+ * numbers we will try in this test. */
+#define NUM_COMBOS 2
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm; /* comm for test */
+
+    /* Num procs for IO and computation. */
+    int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{2, 1}, {1, 2}};
+
+    /* Number of processors that will do IO. */
+    int num_io_procs[NUM_COMBOS] = {2, 1};
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        for (int combo = 0; combo < NUM_COMBOS; combo++)
+        {
+            /* Is the current process a computation task? */
+            int comp_task = my_rank < num_io_procs[combo] ? 0 : 1;
+
+            /* Initialize the IO system. */
+            if ((ret = PIOc_Init_Async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT,
+                                       num_procs[combo], NULL, NULL, NULL, iosysid)))
+                ERR(ERR_INIT);
+
+            for (int c = 0; c < COMPONENT_COUNT; c++)
+                printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
+
+            /* All the netCDF calls are only executed on the computation
+             * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+             * and when the do, they should go straight to finalize. */
+            if (comp_task)
+            {
+                for (int flv = 0; flv < num_flavors; flv++)
+                {
+                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    int my_comp_idx = 0; /* Index in iosysid array. */
+
+                    for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                    {
+                        char iotype_name[NC_MAX_NAME + 1];
+
+                        /* Create a filename. */
+                        if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                            return ret;
+                        sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
+
+                        /* Create sample file. */
+                        printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+                        if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+
+                        /* Check the file for correctness. */
+                        if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+                    }
+                } /* next netcdf flavor */
+
+                /* Finalize the IO system. Only call this from the computation tasks. */
+                printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+                for (int c = 0; c < COMPONENT_COUNT; c++)
+                {
+                    if ((ret = PIOc_finalize(iosysid[c])))
+                        ERR(ret);
+                    printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
+                           iosysid[c]);
+                }
+            } /* endif comp_task */
+
+            /* Wait for everyone to catch up. */
+            printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME);
+            MPI_Barrier(test_comm);
+        } /* next combo */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_async_4proc.c b/src/externals/pio2/tests/cunit/test_async_4proc.c
new file mode 100644
index 00000000000..4f91b0c0eec
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_4proc.c
@@ -0,0 +1,122 @@
+/*
+ * Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ *
+ * This very simple test runs on 4 ranks.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_async_4proc"
+
+/* Number of different combonations of IO and computation processor
+ * numbers we will try in this test. */
+#define NUM_COMBOS 3
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Num procs for IO and computation. */
+    int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{3, 1}, {2, 2}, {1, 3}};
+
+    /* Number of processors that will do IO. */
+    int num_io_procs[NUM_COMBOS] = {3, 2, 1};
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+    
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        for (int combo = 0; combo < NUM_COMBOS; combo++)
+        {
+            /* Is the current process a computation task? */
+            int comp_task = my_rank < num_io_procs[combo] ? 0 : 1;
+
+            /* Initialize the IO system. */
+            if ((ret = PIOc_Init_Async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT,
+                                       num_procs[combo], NULL, NULL, NULL, iosysid)))
+                ERR(ERR_INIT);
+
+            for (int c = 0; c < COMPONENT_COUNT; c++)
+                printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
+
+            /* All the netCDF calls are only executed on the computation
+             * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+             * and when the do, they should go straight to finalize. */
+            if (comp_task)
+            {
+                for (int flv = 0; flv < num_flavors; flv++)
+                {
+                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    int my_comp_idx = 0; /* Index in iosysid array. */
+
+                    for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                    {
+                        char iotype_name[NC_MAX_NAME + 1];
+
+                        /* Create a filename. */
+                        if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                            return ret;
+                        sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
+
+                        /* Create sample file. */
+                        printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+                        if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+
+                        /* Check the file for correctness. */
+                        if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+                    }
+                } /* next netcdf flavor */
+
+                /* Finalize the IO system. Only call this from the computation tasks. */
+                printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+                for (int c = 0; c < COMPONENT_COUNT; c++)
+                {
+                    if ((ret = PIOc_finalize(iosysid[c])))
+                        ERR(ret);
+                    printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
+                           iosysid[c]);
+                }
+            } /* endif comp_task */
+
+            /* Wait for everyone to catch up. */
+            printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME);
+            MPI_Barrier(test_comm);
+        } /* next combo */
+    }/* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_async_simple.c b/src/externals/pio2/tests/cunit/test_async_simple.c
new file mode 100644
index 00000000000..a0852b3e712
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_simple.c
@@ -0,0 +1,122 @@
+/*
+ * Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ *
+ * This very simple test runs on two ranks. One is used for
+ * computation, the other for IO. A sample netCDF file is created and
+ * checked.
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_async_simple
    
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 2
+
+/* The name of this test. */
+#define TEST_NAME "test_async_simple"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* Run simple async test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    int num_procs[COMPONENT_COUNT + 1] = {1, 1}; /* Num procs for IO and computation. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Only do something on TARGET_NTASKS tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Is the current process a computation task? */
+        int comp_task = my_rank < NUM_IO_PROCS ? 0 : 1;
+
+        /* Check for invalid values. */
+        if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                            num_procs, NULL, NULL, NULL, NULL) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, -1,
+                            num_procs, NULL, NULL, NULL, iosysid) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                            NULL, NULL, NULL, NULL, iosysid) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+
+        /* Initialize the IO system. */
+        if ((ret = PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                                   num_procs, NULL, NULL, NULL, iosysid)))
+            ERR(ERR_INIT);
+
+        /* All the netCDF calls are only executed on the computation
+         * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+         * and when the do, they should go straight to finalize. */
+        if (comp_task)
+        {
+            for (int flv = 0; flv < num_flavors; flv++)
+            {
+                int my_comp_idx = my_rank - 1; /* Index in iosysid array. */
+
+                for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                {
+                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    char iotype_name[NC_MAX_NAME + 1];
+
+                    /* Create a filename. */
+                    if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                        return ret;
+                    sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
+
+                    /* Create sample file. */
+                    printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+                    if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                        ERR(ret);
+
+                    /* Check the file for correctness. */
+                    if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                        ERR(ret);
+                }
+            } /* next netcdf flavor */
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+            printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+            for (int c = 0; c < COMPONENT_COUNT; c++)
+            {
+                if ((ret = PIOc_finalize(iosysid[c])))
+                    ERR(ret);
+                printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
+                       iosysid[c]);
+            }
+        } /* endif comp_task */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_common.c b/src/externals/pio2/tests/cunit/test_common.c
new file mode 100644
index 00000000000..e49e093e5f0
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_common.c
@@ -0,0 +1,926 @@
+/*
+ * Common test code for PIO C tests.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of dimensions in the test data. */
+#define NDIM_S1 1
+
+/* The length of our test data. */
+#define DIM_LEN_S1 4
+
+/* The name of the dimension in the netCDF output file. */
+#define FIRST_DIM_NAME_S1 "jojo"
+#define DIM_NAME_S1 "dim_sample_s1"
+
+/* The name of the variable in the netCDF output file. */
+#define FIRST_VAR_NAME_S1 "bill"
+#define VAR_NAME_S1 "var_sample_s1"
+
+/* The number of dimensions in the sample 2 test data. */
+#define NDIM_S2 1
+
+/* The length of our sample 2 test data. */
+#define DIM_LEN_S2 4
+
+/* The name of the dimension in the sample 2 output file. */
+#define FIRST_DIM_NAME_S2 "jojo"
+#define DIM_NAME_S2 "dim_sample_s2"
+
+/* The name of the variable in the sample 2 output file. */
+#define FIRST_VAR_NAME_S2 "bill"
+#define VAR_NAME_S2 "var_sample_s2"
+
+/* The name of the global attribute in the sample 2 output file. */
+#define FIRST_ATT_NAME_S2 "willy_gatt_sample s2"
+#define ATT_NAME_S2 "gatt_sample s2"
+#define SHORT_ATT_NAME_S2 "short_gatt_sample s2"
+#define FLOAT_ATT_NAME_S2 "float_gatt_sample s2"
+#define DOUBLE_ATT_NAME_S2 "double_gatt_sample s2"
+
+/* The value of the global attribute in the sample 2 output file. */
+#define ATT_VALUE_S2 42
+
+/* For when we need 2D. */
+#define NDIM2 2
+
+/* How many flavors of netCDF are available? */
+int
+get_iotypes(int *num_flavors, int *flavors)
+{
+    int num = 0;
+    int fmtidx = 0;
+    int format[NUM_FLAVORS];
+
+#ifdef _PNETCDF
+    num++;
+    format[fmtidx++] = PIO_IOTYPE_PNETCDF;
+#endif
+#ifdef _NETCDF
+    num++;
+    format[fmtidx++] = PIO_IOTYPE_NETCDF;
+#endif
+#ifdef _NETCDF4
+    num += 2;
+    format[fmtidx++] = PIO_IOTYPE_NETCDF4C;
+    format[fmtidx] = PIO_IOTYPE_NETCDF4P;
+#endif
+
+    /* Pass results back to caller. */
+    if (num_flavors)
+        *num_flavors = num;
+    if (flavors)
+        for (int f = 0; f < num; f++)
+            flavors[f] = format[f];
+
+    return PIO_NOERR;
+}
+
+/* Name of each flavor.
+ *
+ * @param iotype the IO type
+ * @param name pointer that will get name of IO type. Must have enough
+ * memory allocated (NC_MAX_NAME + 1 works.)
+ * @returns 0 for success, error code otherwise.
+ * @internal
+ */
+int get_iotype_name(int iotype, char *name)
+{
+    char flavor_name[NUM_FLAVORS][NC_MAX_NAME + 1] = {"pnetcdf", "classic",
+                                                      "serial4", "parallel4"};
+
+    /* Check inputs. */
+    if (!name || iotype < PIO_IOTYPE_PNETCDF || iotype > PIO_IOTYPE_NETCDF4P)
+        return PIO_EINVAL;
+
+    /* Return name of iotype. They are numbered 1-4 in pio.h. */
+    strcpy(name, flavor_name[iotype - 1]);
+
+    return PIO_NOERR;
+}
+
+/* Initalize the test system.
+ *
+ * @param argc argument count from main().
+ * @param argv argument array from main().
+ * @param my_rank pointer that gets this tasks rank.
+ * @param ntasks pointer that gets the number of tasks in WORLD
+ * communicator.
+ * @param target_ntasks the number of tasks this test needs to run.
+ * @param comm a pointer to an MPI communicator that will be created
+ * for this test and contain target_ntasks tasks from WORLD.
+ * @returns 0 for success, error code otherwise.
+*/
+int pio_test_init(int argc, char **argv, int *my_rank, int *ntasks,
+		  int target_ntasks, MPI_Comm *comm)
+{
+    return pio_test_init2(argc,argv, my_rank, ntasks, target_ntasks,
+                          target_ntasks, 3, comm);
+}
+
+/* Initalize the test system.
+ *
+ * @param argc argument count from main().
+ * @param argv argument array from main().
+ * @param my_rank pointer that gets this tasks rank.
+ * @param ntasks pointer that gets the number of tasks in WORLD
+ * communicator.
+ * @param target_ntasks the number of tasks this test needs to run.
+ * @param log_level PIOc_set_log_level() will be called with this value.
+ * @param comm a pointer to an MPI communicator that will be created
+ * for this test and contain target_ntasks tasks from WORLD.
+ * @returns 0 for success, error code otherwise.
+*/
+int pio_test_init2(int argc, char **argv, int *my_rank, int *ntasks,
+                   int min_ntasks, int max_ntasks, int log_level, MPI_Comm *comm)
+{
+    int ret; /* Return value. */
+
+#ifdef TIMING
+    /* Initialize the GPTL timing library. */
+    if ((ret = GPTLinitialize()))
+        return ERR_GPTL;
+#endif
+
+    /* Initialize MPI. */
+    if ((ret = MPI_Init(&argc, &argv)))
+        MPIERR(ret);
+
+    /* Learn my rank and the total number of processors. */
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, my_rank)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, ntasks)))
+        MPIERR(ret);
+    printf("%d has %d tasks\n", *my_rank, *ntasks);
+
+    /* Check that a valid number of processors was specified. */
+    if (*ntasks < min_ntasks)
+    {
+        fprintf(stderr, "ERROR: Number of processors must be at least %d for this test!\n",
+                min_ntasks);
+        return ERR_AWFUL;
+    }
+    else if (*ntasks > max_ntasks)
+    {
+	/* If more tasks are available than we need for this test,
+	 * create a communicator with exactly the number of tasks we
+	 * need. */
+        int color, key;
+        if (*my_rank < max_ntasks)
+        {
+            color = 0;
+	    key = *my_rank;
+        }
+        else
+        {
+            color = 1;
+            key = *my_rank - max_ntasks;
+        }
+	printf("%d splitting comm for test color = %d key = %d\n", *my_rank, color, key);
+        if ((ret = MPI_Comm_split(MPI_COMM_WORLD, color, key, comm)))
+            MPIERR(ret);
+    }
+    else
+    {
+	printf("%d using whole comm for test\n", *my_rank);
+        if ((ret = MPI_Comm_dup(MPI_COMM_WORLD, comm)))
+            MPIERR(ret);
+    }
+
+    /* Turn on logging. */
+    printf("%d setting log level to %d\n", *my_rank, log_level);
+    if ((ret = PIOc_set_log_level(log_level)))
+        return ret;
+    printf("%d done setting log level\n", *my_rank);
+
+    /* Change error handling so we can test inval parameters. */
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Finalize a PIO C test.
+*
+* @param test_comm pointer to the test communicator.
+* @returns 0 for success, error code otherwise.
+*/
+int pio_test_finalize(MPI_Comm *test_comm)
+{
+    int ret = PIO_NOERR; /* Return value. */
+
+    /* Wait for all processes to arrive here. */
+    if (MPI_Barrier(*test_comm))
+        return ERR_MPI;
+
+    /* Free communicator. */
+    if (MPI_Comm_free(test_comm))
+        return ERR_MPI;
+
+    /* Finalize MPI. */
+    MPI_Finalize();
+
+#ifdef TIMING
+    /* Finalize the GPTL timing library. */
+    if ((ret = GPTLfinalize()))
+        return ret;
+#endif
+
+    return ret;
+}
+
+/* Test the inq_format function. */
+int
+test_inq_format(int ncid, int format)
+{
+    int myformat;
+    int ret;
+
+    /* Get the format of an open file. */
+    if ((ret = PIOc_inq_format(ncid, &myformat)))
+        return ret;
+
+    /* Check the result. */
+    if ((format == PIO_IOTYPE_PNETCDF || format == PIO_IOTYPE_NETCDF) && myformat != 1)
+        return ERR_WRONG;
+    else if ((format == PIO_IOTYPE_NETCDF4C || format == PIO_IOTYPE_NETCDF4P) &&
+             myformat != 3)
+        return ERR_WRONG;
+
+    return PIO_NOERR;
+}
+
+/* Test the inq_type function for atomic types. */
+int
+test_inq_type(int ncid, int format)
+{
+#define NUM_TYPES 11
+    char type_name[NC_MAX_NAME + 1];
+    PIO_Offset type_size;
+    nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
+                                NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
+    int type_len[NUM_TYPES] = {1, 1, 2, 4, 4, 8, 1, 2, 4, 8, 8};
+    int max_type = format == PIO_IOTYPE_NETCDF ? NC_DOUBLE : NC_UINT64;
+    int ret;
+
+    /* Check each type size. */
+    for (int i = 0; i < max_type; i++)
+    {
+        if ((ret = PIOc_inq_type(ncid, xtype[i], type_name, &type_size)))
+            return ret;
+        if (type_size != type_len[i])
+            return ERR_AWFUL;
+    }
+
+    return PIO_NOERR;
+}
+
+/* This creates a netCDF sample file in the specified format. */
+int
+create_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid)
+{
+    switch(sample)
+    {
+    case 0:
+        return create_nc_sample_0(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 1:
+        return create_nc_sample_1(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 2:
+        return create_nc_sample_2(iosysid, format, filename, my_rank, ncid);
+        break;
+    }
+    return PIO_EINVAL;
+}
+
+/* This checks a netCDF sample file in the specified format. */
+int
+check_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid)
+{
+    switch(sample)
+    {
+    case 0:
+        return check_nc_sample_0(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 1:
+        return check_nc_sample_1(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 2:
+        return check_nc_sample_2(iosysid, format, filename, my_rank, ncid);
+        break;
+    }
+    return PIO_EINVAL;
+}
+
+/* This creates an empty netCDF file in the specified format. */
+int
+create_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ret;
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* End define mode. */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Test inq_format. */
+    if ((ret = test_inq_format(ncid, format)))
+        return ret;
+
+    /* Test inq_type. */
+    if ((ret = test_inq_type(ncid, format)))
+        return ret;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+        printf("%d closed file ncid = %d\n", my_rank, ncid);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check sample file 1 for correctness. */
+int
+check_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    int ret;
+
+    /* Re-open the file to check it. */
+    printf("%d opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename,
+                             NC_NOWRITE)))
+        return ret;
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != 0 || nvars != 0 || ngatts != 0 || unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        return ret;
+    if (ndims2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        return ret;
+    if (nvars2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        return ret;
+    if (ngatts2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        return ret;
+    if (unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file (again) ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int
+create_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    /* The ncid of the netCDF file. */
+    int ncid;
+
+    /* The ID of the netCDF varable. */
+    int varid;
+
+    /* The ID of the netCDF dimension. */
+    int dimid;
+
+    /* Return code. */
+    int ret;
+
+    /* Start and count arrays for netCDF. */
+    PIO_Offset start[NDIM_S1], count[NDIM_S1] = {0};
+
+    /* The sample data. */
+    int data[DIM_LEN_S1];
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* /\* End define mode, then re-enter it. *\/ */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d calling redef\n", my_rank);
+    if ((ret = PIOc_redef(ncid)))
+        return ret;
+
+    /* Define a dimension. */
+    printf("%d defining dimension %s\n", my_rank, DIM_NAME_S1);
+    if ((ret = PIOc_def_dim(ncid, DIM_NAME_S1, DIM_LEN_S1, &dimid)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    printf("%d defining variable %s\n", my_rank, VAR_NAME_S1);
+    if ((ret = PIOc_def_var(ncid, VAR_NAME_S1, NC_INT, NDIM_S1, &dimid, &varid)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Write some data. For the PIOc_put/get functions, all data must
+     * be on compmaster before the function is called. Only
+     * compmaster's arguments are passed to the async msg handler. All
+     * other computation tasks are ignored. */
+    for (int i = 0; i < DIM_LEN_S1; i++)
+        data[i] = i;
+    printf("%d writing data\n", my_rank);
+    start[0] = 0;
+    count[0] = DIM_LEN_S1;
+    if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
+        return ret;
+
+    /* Test inq_format. */
+    if ((ret = test_inq_format(ncid, format)))
+        return ret;
+
+    /* Test inq_type. */
+    if ((ret = test_inq_type(ncid, format)))
+        return ret;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+        printf("%d closed file ncid = %d\n", my_rank, ncid);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check sample file 1 for correctness. */
+int
+check_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ret;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+    char varname[NC_MAX_NAME + 1];
+    nc_type vartype;
+    int varndims, vardimids, varnatts;
+
+    /* Re-open the file to check it. */
+    printf("%d opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename,
+                             NC_NOWRITE)))
+        return ret;
+
+    /* Try to read the data. */
+    PIO_Offset start[NDIM_S1] = {0}, count[NDIM_S1] = {DIM_LEN_S1};
+    int data_in[DIM_LEN_S1];
+    if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in)))
+        return ret;
+    for (int i = 0; i < DIM_LEN_S1; i++)
+    {
+        printf("%d read data_in[%d] = %d\n", my_rank, i, data_in[i]);
+        if (data_in[i] != i)
+            return ERR_AWFUL;
+    }
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != 1 || nvars != 1 || ngatts != 0 || unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* This should return PIO_NOERR. */
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        return ret;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        return ret;
+    if (ndims2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        return ret;
+    if (nvars2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        return ret;
+    if (ngatts2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        return ret;
+    if (unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* Check out the dimension. */
+    if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen)))
+        return ret;
+    if (strcmp(dimname, DIM_NAME_S1) || dimlen != DIM_LEN_S1)
+        return ERR_WRONG;
+
+    /* Check out the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts)))
+        return ret;
+    if (strcmp(varname, VAR_NAME_S1) || vartype != NC_INT || varndims != NDIM_S1 ||
+        vardimids != 0 || varnatts != 0)
+        return ERR_WRONG;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file (again) ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int
+create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid, varid, dimid;
+    PIO_Offset start[NDIM_S2], count[NDIM_S2] = {0};
+    int data[DIM_LEN_S2];
+    int ret;
+
+    /* Create a netCDF file with one dimension and one variable. */
+    printf("%d creating file %s\n", my_rank, filename);
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* End define mode, then re-enter it. */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    if ((ret = PIOc_redef(ncid)))
+        return ret;
+
+    /* Define a dimension. */
+    char dimname2[NC_MAX_NAME + 1];
+    printf("%d defining dimension %s\n", my_rank, DIM_NAME_S2);
+    if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME_S2, DIM_LEN_S2, &dimid)))
+        return ret;
+    if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+        return ret;
+    if (strcmp(dimname2, FIRST_DIM_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_rename_dim(ncid, 0, DIM_NAME_S2)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    char varname2[NC_MAX_NAME + 1];
+    printf("%d defining variable %s\n", my_rank, VAR_NAME_S2);
+    if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME_S2, NC_INT, NDIM_S2, &dimid, &varid)))
+        return ret;
+    if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+        return ret;
+    if (strcmp(varname2, FIRST_VAR_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_rename_var(ncid, 0, VAR_NAME_S2)))
+        return ret;
+
+    /* Add a global attribute. */
+    printf("%d writing attributes %s\n", my_rank, ATT_NAME_S2);
+    int att_data = ATT_VALUE_S2;
+    short short_att_data = ATT_VALUE_S2;
+    float float_att_data = ATT_VALUE_S2;
+    double double_att_data = ATT_VALUE_S2;
+    char attname2[NC_MAX_NAME + 1];
+    /* Write an att and rename it. */
+    if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NC_INT, 1, &att_data)))
+        return ret;
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2)))
+        return ret;
+    if (strcmp(attname2, FIRST_ATT_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, ATT_NAME_S2)))
+        return ret;
+
+    /* Write an att and delete it. */
+    if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NC_INT, 1, &att_data)))
+        return ret;
+    if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2)))
+        return ret;
+    /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NULL, NULL)) != PIO_ENOTATT) */
+    /* { */
+    /*  printf("ret = %d\n", ret); */
+    /*  return ERR_AWFUL; */
+    /* } */
+
+    /* Write some atts of different types. */
+    if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME_S2, NC_SHORT, 1, &short_att_data)))
+        return ret;
+    if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME_S2, NC_FLOAT, 1, &float_att_data)))
+        return ret;
+    if ((ret = PIOc_put_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME_S2, NC_DOUBLE, 1, &double_att_data)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Write some data. For the PIOc_put/get functions, all data must
+     * be on compmaster before the function is called. Only
+     * compmaster's arguments are passed to the async msg handler. All
+     * other computation tasks are ignored. */
+    for (int i = 0; i < DIM_LEN_S2; i++)
+        data[i] = i;
+    printf("%d writing data\n", my_rank);
+    printf("%d writing data\n", my_rank);
+    start[0] = 0;
+    count[0] = DIM_LEN_S2;
+    if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
+        return ret;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+        printf("%d closed file ncid = %d\n", my_rank, ncid);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check sample file 2 for correctness.
+ *
+ * @param ncidp if NULL, close file, otherwise return ncid of still-open file.
+ * @reaturns 0 for success and error code otherwise.
+ */
+int
+check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ret;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    int dimid2;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+    char dimname2[NC_MAX_NAME + 1];
+    PIO_Offset dimlen2;
+    char varname[NC_MAX_NAME + 1];
+    nc_type vartype;
+    int varndims, vardimids, varnatts;
+    char varname2[NC_MAX_NAME + 1];
+    nc_type vartype2;
+    int varndims2, vardimids2, varnatts2;
+    int varid2;
+    int att_data;
+    short short_att_data;
+    float float_att_data;
+    double double_att_data;
+    nc_type atttype;
+    PIO_Offset attlen;
+    char myattname[NC_MAX_NAME + 1];
+    int myid;
+    PIO_Offset start[NDIM_S2] = {0}, count[NDIM_S2] = {DIM_LEN_S2};
+    int data_in[DIM_LEN_S2];
+
+    /* Re-open the file to check it. */
+    printf("%d opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename, NC_NOWRITE)))
+        return ERR_CHECK;
+
+    /* Try to read the data. */
+    if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in)))
+        return ERR_CHECK;
+    for (int i = 0; i < DIM_LEN_S2; i++)
+    {
+        printf("%d read data_in[%d] = %d\n", my_rank, i, data_in[i]);
+        if (data_in[i] != i)
+            return ERR_AWFUL;
+    }
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ERR_CHECK;
+    if (ndims != 1 || nvars != 1 || ngatts != 4 || unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* This should return PIO_NOERR. */
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        return ERR_CHECK;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        return ERR_CHECK;
+    if (ndims2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        return ERR_CHECK;
+    if (nvars2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        return ERR_CHECK;
+    if (ngatts2 != 4)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        return ERR_CHECK;
+    if (unlimdimid != -1)
+        return ERR_WRONG;
+    /* Should succeed, do nothing. */
+    if ((ret = PIOc_inq_unlimdim(ncid, NULL)))
+        return ERR_CHECK;
+
+    /* Check out the dimension. */
+    if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen)))
+        return ERR_CHECK;
+    if (strcmp(dimname, DIM_NAME_S2) || dimlen != DIM_LEN_S2)
+        return ERR_WRONG;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+        return ERR_CHECK;
+    if (strcmp(dimname2, DIM_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2)))
+        return ERR_CHECK;
+    if (dimlen2 != DIM_LEN_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_dimid(ncid, DIM_NAME_S2, &dimid2)))
+        return ERR_CHECK;
+    if (dimid2 != 0)
+        return ERR_WRONG;
+
+    /* Check out the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts)))
+        return ERR_CHECK;
+    if (strcmp(varname, VAR_NAME_S2) || vartype != NC_INT || varndims != NDIM_S2 ||
+        vardimids != 0 || varnatts != 0)
+        return ERR_WRONG;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+        return ERR_CHECK;
+    if (strcmp(varname2, VAR_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2)))
+        return ERR_CHECK;
+    if (vartype2 != NC_INT)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2)))
+        return ERR_CHECK;
+    if (varndims2 != NDIM_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2)))
+        return ERR_CHECK;
+    if (vardimids2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2)))
+        return ERR_CHECK;
+    if (varnatts2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_varid(ncid, VAR_NAME_S2, &varid2)))
+        return ERR_CHECK;
+    if (varid2 != 0)
+        return ERR_WRONG;
+
+    /* Check out the global attributes. */
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME_S2, &atttype, &attlen)))
+        return ERR_CHECK;
+    if (atttype != NC_INT || attlen != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME_S2, &attlen)))
+        return ERR_CHECK;
+    if (attlen != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, myattname)))
+        return ERR_CHECK;
+    if (strcmp(ATT_NAME_S2, myattname))
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_attid(ncid, NC_GLOBAL, ATT_NAME_S2, &myid)))
+        return ERR_CHECK;
+    if (myid != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME_S2, &att_data)))
+        return ERR_CHECK;
+    printf("%d att_data = %d\n", my_rank, att_data);
+    if (att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, SHORT_ATT_NAME_S2, &atttype, &attlen)))
+        return ERR_CHECK;
+    if (atttype != NC_SHORT || attlen != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME_S2, &short_att_data)))
+        return ERR_CHECK;
+    if (short_att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME_S2, &float_att_data)))
+        return ERR_CHECK;
+    if (float_att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME_S2, &double_att_data)))
+        return ERR_CHECK;
+    if (double_att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file (again) ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ERR_CHECK;
+    }
+
+    return 0;
+}
+
+/* Create the decomposition to divide the 3-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 2 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len_2d an array of length 2 with the dim lengths.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @param pio_type the data type to use for the decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2d,
+                            int *ioid, int pio_type)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 4. */
+    elements_per_pe = dim_len_2d[0] * dim_len_2d[1] / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
diff --git a/src/externals/pio2/tests/cunit/test_darray.c b/src/externals/pio2/tests/cunit/test_darray.c
new file mode 100644
index 00000000000..e53e29bff4e
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray.c
@@ -0,0 +1,375 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, 2/16/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* Test cases relating to PIOc_write_darray_multi(). */
+#define NUM_TEST_CASES_WRT_MULTI 3
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/** 
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int pio_type)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int ncid2;     /* The ncid of the re-opened netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+    PIO_Offset arraylen = 4;
+    void *fillvalue;
+    void *test_data;
+    void *test_data_in;
+    int fillvalue_int = NC_FILL_INT;
+    int test_data_int[arraylen];
+    int test_data_int_in[arraylen];
+    float fillvalue_float = NC_FILL_FLOAT;
+    float test_data_float[arraylen];
+    float test_data_float_in[arraylen];
+    double fillvalue_double = NC_FILL_DOUBLE;
+    double test_data_double[arraylen];
+    double test_data_double_in[arraylen];
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+        test_data_int[f] = my_rank * 10 + f;
+        test_data_float[f] = my_rank * 10 + f + 0.5;
+        test_data_double[f] = my_rank * 100000 + f + 0.5;
+    }
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++) 
+    {
+
+        /* Add a couple of extra tests for the
+         * PIOc_write_darray_multi() function. */
+        for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++)
+        {
+            /* Test with/without providing a fill value to PIOc_write_darray(). */
+            for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+            {
+                /* Create the filename. */
+                sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME,
+                        flavor[fmt], pio_type, test_multi, provide_fill);
+
+                /* Select the fill value and data. */
+                switch (pio_type)
+                {
+                case PIO_INT:
+                    fillvalue = provide_fill ? &fillvalue_int : NULL;
+                    test_data = test_data_int;
+                    test_data_in = test_data_int_in;
+                    break;
+                case PIO_FLOAT:
+                    fillvalue = provide_fill ? &fillvalue_float : NULL;
+                    test_data = test_data_float;
+                    test_data_in = test_data_float_in;
+                    break;
+                case PIO_DOUBLE:
+                    fillvalue = provide_fill ? &fillvalue_double : NULL;
+                    test_data = test_data_double;
+                    test_data_in = test_data_double_in;
+                    break;
+                default:
+                    ERR(ERR_WRONG);
+                }
+
+                /* Create the netCDF output file. */
+                printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename,
+                       flavor[fmt], pio_type);
+                if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                    ERR(ret);
+
+                /* Define netCDF dimensions and variable. */
+                printf("%d Defining netCDF metadata...\n", my_rank);
+                for (int d = 0; d < NDIM; d++)
+                    if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                        ERR(ret);
+
+                /* Define a variable. */
+                if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid)))
+                    ERR(ret);
+
+                /* End define mode. */
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+
+                /* Set the value of the record dimension. */
+                if ((ret = PIOc_setframe(ncid, varid, 0)))
+                    ERR(ret);
+
+                int frame = 0;
+                int flushtodisk = test_multi - 1;
+                if (!test_multi)
+                {
+                    /* These should not work. */
+                    if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray(ncid, varid, ioid + TEST_VAL_42, arraylen, test_data, fillvalue) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray(ncid, varid, ioid, arraylen - 1, test_data, fillvalue) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+
+                    /* Write the data. */
+                    if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+                        ERR(ret);
+                }
+                else
+                {
+                    int varid_big = NC_MAX_VARS + TEST_VAL_42;
+
+                    /* These will not work. */
+                    if (PIOc_write_darray_multi(ncid + TEST_VAL_42, &varid, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray_multi(ncid, NULL, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray_multi(ncid, &varid, ioid + TEST_VAL_42, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray_multi(ncid, &varid, ioid, -1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+                    /* if (PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, NULL, */
+                    /*                             fillvalue, flushtodisk) != PIO_EINVAL) */
+                    /*     ERR(ERR_WRONG); */
+                    if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+                
+                    /* Write the data with the _multi function. */
+                    if ((ret = PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, &frame,
+                                                       fillvalue, flushtodisk)))
+                        ERR(ret);
+                }
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* Reopen the file. */
+                if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_read_darray(ncid2 + TEST_VAL_42, varid, ioid, arraylen,
+                                     test_data_in) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_read_darray(ncid2, varid, ioid + TEST_VAL_42, arraylen,
+                                     test_data_in) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+        
+                /* Read the data. */
+                if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+                    ERR(ret);
+
+                /* Check the results. */
+                for (int f = 0; f < arraylen; f++)
+                {
+                    switch (pio_type)
+                    {
+                    case PIO_INT:
+                        if (test_data_int_in[f] != test_data_int[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_FLOAT:
+                        if (test_data_float_in[f] != test_data_float[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_DOUBLE:
+                        if (test_data_double_in[f] != test_data_double[f])
+                            return ERR_WRONG;
+                        break;
+                    default:
+                        ERR(ERR_WRONG);
+                    }
+                }
+
+                /* Try to write, but it won't work, because we opened file read-only. */
+                if (!test_multi)
+                {
+                    if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM)
+                        ERR(ERR_WRONG);
+                }
+                else
+                {
+                    if (PIOc_write_darray_multi(ncid2, &varid, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EPERM)
+                        ERR(ERR_WRONG);
+                }
+        
+                /* Close the netCDF file. */
+                printf("%d Closing the sample data file...\n", my_rank);
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+            } /* next fillvalue test case */
+        } /* next test multi */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests. 
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise. 
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm)
+{
+#define NUM_TYPES_TO_TEST 3
+    int ioid;
+    char filename[NC_MAX_NAME + 1];
+    int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+    {        
+        /* This will be our file name for writing out decompositions. */
+        sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank,
+                *flavor, pio_type[t]);
+
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+                                           &ioid, pio_type[t])))
+        return ret;
+
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t])))
+            return ret;
+    
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+        
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+        /* Initialize the PIO IO system. This specifies how
+         * many and which processors are involved in I/O. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                       ioproc_start, rearranger[r], &iosysid)))
+            return ret;
+
+        /* Run tests. */
+        printf("%d Running tests...\n", my_rank);
+        if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm)))
+            return ret;
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            return ret;
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_1d.c b/src/externals/pio2/tests/cunit/test_darray_1d.c
new file mode 100644
index 00000000000..01b894a8749
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_1d.c
@@ -0,0 +1,620 @@
+/*
+ * Tests for PIO distributed arrays. This test uses 1 dimension,
+ * everything very simple. ;-)
+ *
+ * Ed Hartnett, 2/27/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_1d"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 1
+
+/* The length of our sample data along the dimension. */
+#define DIM_LEN 8
+
+/* This is the length of the map for each task. */
+#define EXPECTED_MAPLEN 2
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "RedShirtSurvival"
+
+/* The dimension names. */
+#define DIM_NAME "episode"
+#define DIM_NAME_2 "phaser_draws"
+
+/* Create a 1D decomposition.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @param pio_type the type that will be used for basetype.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_1d(int ntasks, int my_rank, int iosysid, int pio_type, int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    int dim_len_1d[NDIM] = {DIM_LEN};
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 2. */
+    elements_per_pe = DIM_LEN / ntasks;
+
+    PIO_Offset compdof[elements_per_pe];
+
+    /* Don't forget to add 1! */
+    compdof[0] = my_rank + 1;
+
+    /* This means fill value will be used here. */
+    compdof[1] = 0;
+
+    /* Create the PIO decomposition for this test. */
+    if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM, dim_len_1d, elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    return 0;
+}
+
+/**
+ * Test fill values and darrays.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param pio_type the type of the data.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int *flavor,
+                     int my_rank, MPI_Comm test_comm)
+{
+#define NUM_FILLVALUE_PRESENT_TESTS 2
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimid;     /* The dimension ID. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    PIO_Offset arraylen = 2;
+    void *test_data;
+    void *fillvalue;
+    void *test_data_in;
+    void *expected_in;
+    PIO_Offset type_size;             /* Size of the data type. */
+    float my_float_rank = my_rank;    /* my_rank in a float. */
+    double my_double_rank = my_rank;  /* my_rank in a double. */
+    int int_fill = NC_FILL_INT;
+    float float_fill = NC_FILL_FLOAT;
+    double double_fill = NC_FILL_DOUBLE;
+    void *bufr;
+    int ret;                        /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        for (int with_fillvalue = 0; with_fillvalue < NUM_FILLVALUE_PRESENT_TESTS; with_fillvalue++)
+        {
+            /* Create the filename. */
+            sprintf(filename, "data_%s_iotype_%d_pio_type_%d_with_fillvalue_%d.nc", TEST_NAME, flavor[fmt],
+                    pio_type, with_fillvalue);
+
+            /* Create the netCDF output file. */
+            printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+                   flavor[fmt]);
+            if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                ERR(ret);
+
+            /* Turn on fill mode. */
+            if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+                ERR(ret);
+
+            /* Define netCDF dimensions and variable. */
+            if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)))
+                ERR(ret);
+
+            /* Define a variable. */
+            if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, &dimid, &varid)))
+                ERR(ret);
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                ERR(ret);
+
+            /* Get the size of the type. */
+            if ((ret = PIOc_inq_type(ncid, pio_type, NULL, &type_size)))
+                return ret;
+
+            /* Initialize some data. */
+            int int_test_data[2] = {my_rank, my_rank};
+            float float_test_data[2] = {my_rank, my_rank};
+            double double_test_data[2] = {my_rank, my_rank};
+            switch (pio_type)
+            {
+            case PIO_INT:
+                test_data = int_test_data;
+                fillvalue = with_fillvalue ? &int_fill : NULL;
+                expected_in = &my_rank;
+                break;
+            case PIO_FLOAT:
+                test_data = float_test_data;
+                fillvalue = with_fillvalue ? &float_fill : NULL;
+                expected_in = &my_float_rank;
+                break;
+            case PIO_DOUBLE:
+                test_data = double_test_data;
+                fillvalue = with_fillvalue ? &double_fill : NULL;
+                expected_in = &my_double_rank;
+                break;
+            default:
+                return ERR_WRONG;
+            }
+
+            /* Write the data. Our test_data contains only one real value
+             * (instead of 2, as indicated by arraylen), but due to the
+             * decomposition, only the first value is used in the
+             * output. */
+            if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data,
+                                         fillvalue)))
+                ERR(ret);
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+            /* Reopen the file. */
+            if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, PIO_NOWRITE)))
+                ERR(ret);
+
+            /* Allocate space for data. */
+            if (!(test_data_in = malloc(type_size * arraylen)))
+                ERR(PIO_ENOMEM);
+
+            /* Read the data. */
+            if ((ret = PIOc_read_darray(ncid, varid, ioid, arraylen, test_data_in)))
+                ERR(ret);
+
+            /* Check the (first) result. */
+            if (memcmp(test_data_in, expected_in, type_size))
+                return ERR_WRONG;
+
+            /* Free resources. */
+            free(test_data_in);
+
+            /* Get a buffer big enough to hold the global array. */
+            if (!(bufr = malloc(DIM_LEN * type_size)))
+                return PIO_ENOMEM;
+
+            /* Get the whole array with good old get_var(). */
+            if ((ret = PIOc_get_var(ncid, varid, bufr)))
+                return ret;
+
+            /* Check the results. The first four values are 0, 1, 2, 3,
+             * and the rest are the default fill value of the type. */
+            for (int e = 0; e < DIM_LEN; e++)
+            {
+                switch (pio_type)
+                {
+                case PIO_INT:
+                    if (((int *)bufr)[e] != (e < 4 ? e : NC_FILL_INT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_FLOAT:
+                    if (((float *)bufr)[e] != (e < 4 ? e : NC_FILL_FLOAT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_DOUBLE:
+                    if (((double *)bufr)[e] != (e < 4 ? e : NC_FILL_DOUBLE))
+                        return ERR_WRONG;
+                    break;
+                default:
+                    return ERR_WRONG;
+                }
+            }
+
+            /* Release buffer. */
+            free(bufr);
+
+            /* Close the netCDF file. */
+            printf("%d Closing the sample data file...\n", my_rank);
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+        } /* with_fillvalue */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test fill values and darrays with an unlimited dim.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param pio_type the type of the data.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors,
+                           int *flavor, int my_rank, MPI_Comm test_comm)
+{
+#define NDIM2 2
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimid[NDIM2];     /* The dimension ID. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    PIO_Offset arraylen = 2;
+    void *test_data;
+    void *fillvalue;
+    void *test_data_in;
+    void *expected_in;
+    PIO_Offset type_size;             /* Size of the data type. */
+    float my_float_rank = my_rank;    /* my_rank in a float. */
+    double my_double_rank = my_rank;  /* my_rank in a double. */
+    int int_fill = NC_FILL_INT;
+    float float_fill = NC_FILL_FLOAT;
+    double double_fill = NC_FILL_DOUBLE;
+    void *bufr;
+    int ret;                        /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "data_%s_iotype_%d_pio_type_%d_unlim.nc", TEST_NAME, flavor[fmt],
+                pio_type);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+               flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Turn on fill mode. */
+        if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+            ERR(ret);
+
+        /* Define netCDF dimensions. */
+        if ((ret = PIOc_def_dim(ncid, DIM_NAME, NC_UNLIMITED, &dimid[0])))
+            ERR(ret);
+        if ((ret = PIOc_def_dim(ncid, DIM_NAME_2, DIM_LEN, &dimid[1])))
+            ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM2, dimid, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Get the size of the type. */
+        if ((ret = PIOc_inq_type(ncid, pio_type, NULL, &type_size)))
+            return ret;
+
+        /* Initialize some data. */
+        int int_test_data[2] = {my_rank, my_rank};
+        float float_test_data[2] = {my_rank, my_rank};
+        double double_test_data[2] = {my_rank, my_rank};
+        switch (pio_type)
+        {
+        case PIO_INT:
+            test_data = int_test_data;
+            fillvalue = &int_fill;
+            expected_in = &my_rank;
+            break;
+        case PIO_FLOAT:
+            test_data = float_test_data;
+            fillvalue = &float_fill;
+            expected_in = &my_float_rank;
+            break;
+        case PIO_DOUBLE:
+            test_data = double_test_data;
+            fillvalue = &double_fill;
+            expected_in = &my_double_rank;
+            break;
+        default:
+            return ERR_WRONG;
+        }
+
+        /* Set the record number for the unlimited dimension. */
+        if ((ret = PIOc_setframe(ncid, varid, 0)))
+            ERR(ret);
+
+        /* Write the data. Our test_data contains only one real value
+         * (instead of 2, as indicated by arraylen), but due to the
+         * decomposition, only the first value is used in the
+         * output. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Set the record number for the unlimited dimension. */
+        if ((ret = PIOc_setframe(ncid, varid, 1)))
+            ERR(ret);
+
+        /* Write the data. Our test_data contains only one real value
+         * (instead of 2, as indicated by arraylen), but due to the
+         * decomposition, only the first value is used in the
+         * output. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Allocate space for data. */
+        if (!(test_data_in = malloc(type_size * arraylen)))
+            ERR(PIO_ENOMEM);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid, varid, ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the (first) result. */
+        if (memcmp(test_data_in, expected_in, type_size))
+            return ERR_WRONG;
+
+        /* Free resources. */
+        free(test_data_in);
+
+        /* Get a buffer big enough to hold the global array. */
+        if (!(bufr = malloc(DIM_LEN * type_size * 2)))
+            return PIO_ENOMEM;
+
+        /* Get the whole array with good old get_var(). */
+        if ((ret = PIOc_get_var(ncid, varid, bufr)))
+            return ret;
+
+        /* Check the results. The first four values in each record are
+         * 0, 1, 2, 3, and the rest are the default fill value of the
+         * type. There are two records. */
+        for (int e = 0; e < DIM_LEN * 2; e++)
+        {
+            switch (pio_type)
+            {
+            case PIO_INT:
+                if (((int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT))
+                    return ERR_WRONG;
+                break;
+            case PIO_FLOAT:
+                if (((float *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_FLOAT))
+                    return ERR_WRONG;
+                break;
+            case PIO_DOUBLE:
+                if (((double *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_DOUBLE))
+                    return ERR_WRONG;
+                break;
+            default:
+                return ERR_WRONG;
+            }
+        }
+
+        /* Release buffer. */
+        free(bufr);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type involved in this decompositon.
+ * @param rearranger the rearranger in use.
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           int pio_type, int rearranger, MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int ioid2;             /* ID for decomposition we will create from file. */
+    char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+    char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int ret;              /* Return code. */
+
+    /* Use PIO to create the decomp file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
+
+        printf("writing decomp file %s\n", filename);
+        if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, test_comm, NULL,
+                                        NULL, 0)))
+            return ret;
+
+        /* Read the data. */
+        printf("reading decomp file %s\n", filename);
+        if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, pio_type,
+                                       title_in, history_in, &fortran_order_in)))
+            return ret;
+
+        /* Check the results. */
+        {
+            iosystem_desc_t *ios;
+            io_desc_t *iodesc;
+            int expected_basetype;
+
+            switch (pio_type)
+            {
+            case PIO_INT:
+                expected_basetype = MPI_INT;
+                break;
+            case PIO_FLOAT:
+                expected_basetype = MPI_FLOAT;
+                break;
+            case PIO_DOUBLE:
+                expected_basetype = MPI_DOUBLE;
+                break;
+            default:
+                return ERR_WRONG;
+            }
+
+            /* Get the IO system info. */
+            if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* Get the IO desc, which describes the decomposition. */
+            if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+            if (iodesc->ioid != ioid2 || iodesc->maplen != EXPECTED_MAPLEN || iodesc->ndims != NDIM)
+                return ERR_WRONG;
+            /* if (iodesc->nrecvs != 1) */
+            /*     return ERR_WRONG; */
+            /* if (iodesc->num_aiotasks != TARGET_NTASKS) */
+            /*     return ERR_WRONG; */
+            printf("iodesc->nrecvs = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, iodesc->num_aiotasks);
+            if (iodesc->ndof != EXPECTED_MAPLEN)
+                return ERR_WRONG;
+            if (iodesc->rearranger != rearranger || iodesc->maxregions != 1)
+                return ERR_WRONG;
+            if (!iodesc->needsfill || iodesc->basetype != expected_basetype)
+                return ERR_WRONG;
+            /* Don't forget to add 1!! */
+            if (iodesc->map[0] != my_rank + 1 || iodesc->map[1] != 0)
+                return ERR_WRONG;
+            if (iodesc->dimlen[0] != DIM_LEN)
+                return ERR_WRONG;
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+#define NUM_TYPES_TO_TEST 3
+    int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioid;     /* Decomposition ID. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            /* Initialize the PIO IO system. This specifies how many and
+             * which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+
+            /* Run tests for each data type. */
+            for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+            {
+                /* Decompose the data over the tasks. */
+                if ((ret = create_decomposition_1d(TARGET_NTASKS, my_rank, iosysid, test_type[t],
+                                                   &ioid)))
+                    return ret;
+
+                /* Test decomposition read/write. */
+                if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank,
+                                                  test_type[t], rearranger[r], test_comm)))
+                    return ret;
+
+                /* Run tests. */
+                if ((ret = test_darray_fill(iosysid, ioid, test_type[t], num_flavors, flavor,
+                                            my_rank, test_comm)))
+                    return ret;
+
+                /* Run tests. */
+                if ((ret = test_darray_fill_unlim(iosysid, ioid, test_type[t], num_flavors,
+                                                  flavor, my_rank, test_comm)))
+                    return ret;
+
+                /* Free the PIO decomposition. */
+                if ((ret = PIOc_freedecomp(iosysid, ioid)))
+                    ERR(ret);
+            }
+
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+        } /* next rearranger */
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_3d.c b/src/externals/pio2/tests/cunit/test_darray_3d.c
new file mode 100644
index 00000000000..07f2cd4e303
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_3d.c
@@ -0,0 +1,423 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, 2/21/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_3d"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 4
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM3 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+#define Z_DIM_LEN 4
+
+/* This is the length of the map for each task. */
+#define EXPECTED_MAPLEN 16
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y", "z"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+
+/* Run test for each of the rearrangers. */
+#define NUM_REARRANGERS_TO_TEST 2
+
+/* Create the decomposition to divide the 4-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 3 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int dim_len_3d[NDIM3] = {X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 4. */
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe,
+                                compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/**
+ * Test the darray functionality. Create a netCDF file with 4
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray().
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int provide_fill)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int ncid2;     /* The ncid of the re-opened netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+    PIO_Offset arraylen = 16;
+    int int_fillvalue = NC_FILL_INT;
+    void *fillvalue = NULL;
+    int test_data[arraylen];
+    int test_data2[arraylen];
+    int test_data_in[arraylen];
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+        test_data[f] = my_rank * 10 + f;
+        test_data2[f] = 2 * (my_rank * 10 + f);
+    }
+
+    /* Are we providing a fill value? */
+    if (provide_fill)
+        fillvalue = &int_fillvalue;
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+               flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Turn on fill mode. */
+        if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("%d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Set the value of the record dimension. */
+        if ((ret = PIOc_setframe(ncid, varid, 0)))
+            ERR(ret);
+
+        /* Write the data. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Set the value of the record dimension to the second record. */
+        if ((ret = PIOc_setframe(ncid, varid, 1)))
+            ERR(ret);
+
+        /* Write the data for the second record. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data2, fillvalue)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the results. */
+        for (int f = 0; f < arraylen; f++)
+            if (test_data_in[f] != test_data[f])
+                return ERR_WRONG;
+
+        /* Set the value of the record dimension to the second record. */
+        if ((ret = PIOc_setframe(ncid2, varid, 1)))
+            ERR(ret);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the results. */
+        for (int f = 0; f < arraylen; f++)
+            if (test_data_in[f] != test_data2[f])
+                return ERR_WRONG;
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           int rearranger, MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int ioid2;             /* ID for decomposition we will create from file. */
+    char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+    char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int ret;              /* Return code. */
+
+    /* Use PIO to create the decomp file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
+
+        printf("writing decomp file %s\n", filename);
+        if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, test_comm, NULL,
+                                        NULL, 0)))
+            return ret;
+
+        /* Read the data. */
+        printf("reading decomp file %s\n", filename);
+        if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT,
+                                       title_in, history_in, &fortran_order_in)))
+            return ret;
+
+        /* Check the results. */
+        {
+            iosystem_desc_t *ios;
+            io_desc_t *iodesc;
+
+            /* Get the IO system info. */
+            if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* Get the IO desc, which describes the decomposition. */
+            if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+            if (iodesc->ioid != ioid2 || iodesc->maplen != EXPECTED_MAPLEN || iodesc->ndims != NDIM3 ||
+                iodesc->ndof != EXPECTED_MAPLEN)
+                return ERR_WRONG;
+            if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 ||
+                iodesc->needsfill || iodesc->basetype != MPI_INT)
+                return ERR_WRONG;
+            for (int e = 0; e < iodesc->maplen; e++)
+                if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1)
+                    return ERR_WRONG;
+            if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN ||
+                iodesc->dimlen[2] != Z_DIM_LEN)
+                return ERR_WRONG;
+            if (rearranger == PIO_REARR_SUBSET)
+            {
+                if (iodesc->nrecvs != 1  || iodesc->num_aiotasks != TARGET_NTASKS)
+                    return ERR_WRONG;
+            }
+            else
+            {
+                /* I haven't figured out yet what these should be for
+                 * the box rearranger. */
+                printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs,
+                       iodesc->num_aiotasks);
+            }
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests.
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    int rearranger, MPI_Comm test_comm)
+{
+    int ioid;
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, &ioid)))
+        return ret;
+
+    /* Test decomposition read/write. */
+    if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank,
+                                      rearranger, test_comm)))
+        return ret;
+
+    /* Test with/without providing a fill value to PIOc_write_darray(). */
+    for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+    {
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, provide_fill)))
+            return ret;
+    }
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+
+            /* Run tests. */
+            printf("%d Running tests...\n", my_rank);
+            if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank,
+                                       rearranger[r], test_comm)))
+                return ret;
+
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar.c b/src/externals/pio2/tests/cunit/test_darray_multivar.c
new file mode 100644
index 00000000000..c4b91c32ae2
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_multivar.c
@@ -0,0 +1,407 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, 2/16/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_multivar"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variables in the netCDF output files. */
+#define VAR_NAME_1 "STICKS"
+#define VAR_NAME_2 "NIX"
+#define VAR_NAME_3 "HICK"
+#define VAR_NAME_4 "PIX"
+
+/* Number of variables in the test file. */
+#define NUM_VAR 4
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* The var names. */
+char var_name[NUM_VAR][PIO_MAX_NAME + 1] = {"STICKS", "NIX", "HICK", "PIX"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/** 
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 4 variables, and use darray to write to one of them.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @param test_comm the communicator that is running this test.
+ * @param rearranger the rearranger in use for this test.
+ * @param use_fill 1 if fill mode should be set.
+ * @param use_default 1 if default fill values should be used
+ * (ignored if use_fill is 0).
+ * @returns 0 for success, error code otherwise.
+*/
+int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                 int pio_type, MPI_Comm test_comm, int rearranger, int use_fill,
+                 int use_default)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];     /* The dimension IDs. */
+    int ncid;             /* The ncid of the netCDF file. */
+    int ncid2;            /* The ncid of the re-opened netCDF file. */
+    int varid[NUM_VAR];   /* The IDs of the netCDF varables. */
+    PIO_Offset arraylen = 4;
+    void *fillvalue;
+    void *test_data;
+    void *test_data_in;
+    int fillvalue_int = NC_FILL_INT;
+    int custom_fillvalue_int = -TEST_VAL_42;
+    int test_data_int[arraylen];
+    int test_data_int_in[arraylen];
+    float fillvalue_float = NC_FILL_FLOAT;
+    float custom_fillvalue_float = -TEST_VAL_42;
+    float test_data_float[arraylen];
+    float test_data_float_in[arraylen];
+    double fillvalue_double = NC_FILL_DOUBLE;
+    double custom_fillvalue_double = (-TEST_VAL_42 * 100);
+    double test_data_double[arraylen];
+    double test_data_double_in[arraylen];
+    int ret;       /* Return code. */
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+        test_data_int[f] = my_rank * 10 + f;
+        test_data_float[f] = my_rank * 10 + f + 0.5;
+        test_data_double[f] = my_rank * 100000 + f + 0.5;
+    }
+
+    /* Select the fill value and data. */
+    switch (pio_type)
+    {
+    case PIO_INT:
+        fillvalue = use_default ? &fillvalue_int : &custom_fillvalue_int;
+        test_data = test_data_int;
+        test_data_in = test_data_int_in;
+        break;
+    case PIO_FLOAT:
+        fillvalue = use_default ? &fillvalue_float : &custom_fillvalue_float;
+        test_data = test_data_float;
+        test_data_in = test_data_float_in;
+        break;
+    case PIO_DOUBLE:
+        fillvalue = use_default ? &fillvalue_double : &custom_fillvalue_double;
+        test_data = test_data_double;
+        test_data_in = test_data_double_in;
+        break;
+    default:
+        ERR(ERR_WRONG);
+    }
+
+    /* Try in pnetcdf only. This code demonstrates that pnetcdf fill
+     * values do work. But not for PIO, so we have a bug somewhere. */
+    {
+#ifdef _PNETCDF
+        int ncid;
+        int varid;
+        int dimid;
+        char test_filename[] = "pnetcdf_test.nc";
+        int ret;
+
+        if ((ret = ncmpi_create(test_comm, test_filename, NC_CLOBBER, MPI_INFO_NULL, &ncid)))
+            return ret;
+        if ((ret = ncmpi_set_fill(ncid, NC_FILL, NULL)))
+            return ret;            
+        if ((ret = ncmpi_def_dim(ncid, "dim_name", 5, &dimid)))
+            return ret;
+        if ((ret = ncmpi_def_var(ncid, "dim_name", NC_INT, 1, &dimid, &varid)))
+            return ret;
+        if ((ret = ncmpi_enddef(ncid)))
+            return ret;
+        if ((ret = ncmpi_close(ncid)))
+            return ret;
+
+        /* Reopen and check. */
+        if ((ret = ncmpi_open(test_comm, test_filename, NC_NOWRITE, MPI_INFO_NULL, &ncid)))
+            return ret;
+        int datum;
+        MPI_Offset start[1] = {0};
+        ret = ncmpi_get_var1_int(ncid, varid, start, &datum);
+        printf("datum ret = %d\n", ret);
+
+        /* Not sure why this doesn't work. */
+        /* if ((ret = ncmpi_get_var1_int(ncid, varid, start, &datum))) */
+        /*     return ret; */
+        printf("datum = %d\n", datum);
+        if ((ret = ncmpi_close(ncid)))
+            return ret;
+#endif /* _PNETCDF */
+    }
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++) 
+    {
+        /* Create the filename. */
+        sprintf(filename, "data_%s_iotype_%d_pio_type_%d_use_fill_%d_default_fill_%d.nc",
+                TEST_NAME, flavor[fmt], pio_type, use_fill, use_default);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename,
+               flavor[fmt], pio_type);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Turn on fill mode if desired. */
+        if (use_fill)
+            if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+                ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("%d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+
+        /* Define the variables. */
+        for (int v = 0; v < NUM_VAR; v++)
+        {
+            if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v])))
+                ERR(ret);
+            if (use_fill && !use_default)
+                if ((ret = PIOc_def_var_fill(ncid, varid[v], NC_FILL, fillvalue)))
+                    ERR(ret);
+        }
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Set the value of the record dimension. */
+        if ((ret = PIOc_setframe(ncid, varid[0], 0)))
+            ERR(ret);
+
+        /* Write the data. */
+        if ((ret = PIOc_write_darray(ncid, varid[0], ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid2, varid[0], ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the results. */
+        for (int f = 0; f < arraylen; f++)
+        {
+            switch (pio_type)
+            {
+            case PIO_INT:
+                if (test_data_int_in[f] != test_data_int[f])
+                    return ERR_WRONG;
+                break;
+            case PIO_FLOAT:
+                if (test_data_float_in[f] != test_data_float[f])
+                    return ERR_WRONG;
+                break;
+            case PIO_DOUBLE:
+                if (test_data_double_in[f] != test_data_double[f])
+                    return ERR_WRONG;
+                break;
+            default:
+                ERR(ERR_WRONG);
+            }
+        }
+
+        /* If fill mode is in use the other vars should have fill values. */
+        if (use_fill && flavor[fmt] != PIO_IOTYPE_PNETCDF)
+        {
+            /* Read the data. */
+            if ((ret = PIOc_read_darray(ncid2, varid[1], ioid, arraylen, test_data_in)))
+                ERR(ret);
+            
+            /* Check the results. */
+            for (int f = 0; f < arraylen; f++)
+            {
+                switch (pio_type)
+                {
+                case PIO_INT:
+                    if (test_data_int_in[f] != (use_default ? NC_FILL_INT : custom_fillvalue_int))
+                        return ERR_WRONG;
+                    break;
+                case PIO_FLOAT:
+                    if (test_data_float_in[f] != (use_default ? NC_FILL_FLOAT : custom_fillvalue_float))
+                        return ERR_WRONG;
+                    break;
+                case PIO_DOUBLE:
+                    if (test_data_double_in[f] != (use_default ? NC_FILL_DOUBLE : custom_fillvalue_double))
+                        return ERR_WRONG;
+                    break;
+                default:
+                    ERR(ERR_WRONG);
+                }
+            }
+        }
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests. 
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @param rearranger the rearranger in use in this test.
+ * @returns 0 for success, error code otherwise. 
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm, int rearranger)
+{
+#define NUM_FILL_TESTS 3
+#define NUM_TYPES_TO_TEST 3
+    int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+    int ioid;
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+    {        
+        int use_fill = 0;
+        int use_default = 0;
+        
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+                                           &ioid, pio_type[t])))
+        return ret;
+
+        /* Run the different combinations of use_fill and use_default. */
+        for (int f = 0; f < NUM_FILL_TESTS; f++)
+        {
+            /* Set flags for this test case. */
+            if (f == 1)
+                use_fill++;
+            if (f == 2)
+                use_default++;
+            
+            /* Run a simple darray test. */
+            if ((ret = test_3_empty(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t],
+                                    test_comm, rearranger, use_fill, use_default)))
+                return ret;
+        }
+        
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS 2
+    int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors;         /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm;      /* A communicator for this test. */
+    int ret;                 /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, MIN_NTASKS,
+                              3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;              /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;                  /* Return code. */
+        
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        /* Test for both arrangers. */
+        for (int r = 0; r < NUM_REARRANGERS; r++)
+        {
+
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+            
+            /* Run tests. */
+            printf("%d Running tests...\n", my_rank);
+            if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm,
+                                       rearranger[r])))
+                return ret;
+            
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+        }
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_decomps.c b/src/externals/pio2/tests/cunit/test_decomps.c
new file mode 100644
index 00000000000..032d10de7b0
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_decomps.c
@@ -0,0 +1,421 @@
+/*
+ * Tests for PIO data decompositons.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_decomps"
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* In decompositions we specify with 1 less dimension. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 1
+
+/* Files of decompositions. */
+#define DECOMP_FILE "decomp.txt"
+#define DECOMP_BC_FILE "decomp.txt"
+
+/* Used when initializing PIO. */
+#define STRIDE1 1
+#define STRIDE2 2
+#define BASE0 0
+#define BASE1 1
+#define NUM_IO1 1
+#define NUM_IO2 2
+#define NUM_IO4 4
+#define REARRANGER 2
+
+/**
+ * Test some decomposition functions. 
+ *
+ * @param iosysid the IO system ID.
+ * @param my_rank the 0-based rank of this task.
+ * @param test_comm communicator that includes all tasks paticipating in test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_decomp1(int iosysid, int my_rank, MPI_Comm test_comm)
+{
+    int ioid;                   /* The decomposition ID. */
+    PIO_Offset elements_per_pe; /* Array index per processing unit. */
+    PIO_Offset *compdof;        /* The decomposition mapping. */
+    int slice_dimlen[2];
+    int bad_slice_dimlen[2];    /* Invalid values. */
+    int ndims;
+    int *gdims;
+    PIO_Offset fmaplen;
+    PIO_Offset *map;
+    int ret;
+    
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    slice_dimlen[0] = X_DIM_LEN;
+    slice_dimlen[1] = Y_DIM_LEN;
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN / TARGET_NTASKS;
+
+    /* The compdof array contains a mapping for this task into the
+     * global data array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* These should not work. */
+    bad_slice_dimlen[1] = 0;
+    if (PIOc_InitDecomp(iosysid + TEST_VAL_42, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                        compdof, &ioid, NULL, NULL, NULL) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, bad_slice_dimlen, (PIO_Offset)elements_per_pe,
+                        compdof, &ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, NULL, (PIO_Offset)elements_per_pe,
+                        compdof, &ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                        NULL, &ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                        compdof, NULL, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+        
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition...\n", my_rank);
+    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                               compdof, &ioid, NULL, NULL, NULL)))
+        return ret;
+    free(compdof);
+
+    /* These should not work. */
+    if (PIOc_write_decomp(DECOMP_FILE, iosysid + TEST_VAL_42, ioid, test_comm) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_write_decomp(DECOMP_FILE, iosysid, ioid + TEST_VAL_42, test_comm) != PIO_EBADID)
+        return ERR_WRONG;
+
+    /* Write the decomp file. */
+    if ((ret = PIOc_write_decomp(DECOMP_FILE, iosysid, ioid, test_comm)))
+        return ret;
+
+    /* These should not work. */
+    if (PIOc_readmap(NULL, &ndims, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, NULL, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, &ndims, NULL, &fmaplen, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, &ndims, (int **)&gdims, NULL, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, &ndims, (int **)&gdims, &fmaplen, NULL, test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Read the decomp file and check results. */
+    if ((ret = PIOc_readmap(DECOMP_FILE, &ndims, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                            test_comm)))
+        return ret;
+    printf("ndims = %d fmaplen = %lld\n", ndims, fmaplen);
+    if (ndims != 2 || fmaplen != 4)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims; d++)
+    {
+        printf("gdims[%d] = %d\n", d, gdims[d]);
+    }
+    for (int m = 0; m < fmaplen; m++)
+    {
+        printf("map[%d] = %lld\n", m, map[m]);
+    }
+
+    free(map);
+    free(gdims);
+
+    /* These should not work. */
+    if (PIOc_freedecomp(iosysid + TEST_VAL_42, ioid) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_freedecomp(iosysid, ioid + TEST_VAL_42) != PIO_EBADID)
+        return ERR_WRONG;
+        
+    /* Free the PIO decomposition. */
+    printf("%d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+        
+    return 0;
+}
+
+/**
+ * Test PIOc_InitDecomp_bc(). 
+ *
+ * @param iosysid the IO system ID.
+ * @param my_rank the 0-based rank of this task.
+ * @param test_comm communicator that includes all tasks paticipating in test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_decomp_bc(int iosysid, int my_rank, MPI_Comm test_comm)
+{
+    int ioid;                   /* The decomposition ID. */
+    long int start[NDIM2];
+    long int count[NDIM2];
+    long int bad_count[NDIM2] = {-1, 0};
+    long int bad_start[NDIM2] = {-1, 0};
+    int ndims;
+    int *gdims;
+    PIO_Offset fmaplen;
+    PIO_Offset *map;
+    int slice_dimlen[NDIM2];
+    int ret;
+    
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    start[0] = my_rank;
+    start[1] = 0;
+    count[0] = 1;
+    count[1] = 4;
+    slice_dimlen[0] = X_DIM_LEN;
+    slice_dimlen[1] = Y_DIM_LEN;
+
+    /* These should not work. */
+    if (PIOc_InitDecomp_bc(iosysid + TEST_VAL_42, PIO_FLOAT, 2, slice_dimlen, start, count, &ioid) != PIO_EBADID)
+        return ERR_WRONG;
+    printf("ret = %d\n", PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, NULL, start, count, &ioid));
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, NULL, start, count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, NULL, count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, NULL, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, count, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, bad_start, count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, bad_count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+        
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition...\n", my_rank);
+    if ((ret = PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, count, &ioid)))
+        return ret;
+
+    /* Write the decomp file. */
+    if ((ret = PIOc_write_decomp(DECOMP_BC_FILE, iosysid, ioid, test_comm)))
+        return ret;
+
+    /* Read the decomp file and check results. */
+    if ((ret = PIOc_readmap(DECOMP_BC_FILE, &ndims, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                            test_comm)))
+        return ret;
+    printf("ndims = %d fmaplen = %lld\n", ndims, fmaplen);
+    if (ndims != 2 || fmaplen != 4)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims; d++)
+    {
+        printf("gdims[%d] = %d\n", d, gdims[d]);
+    }
+    for (int m = 0; m < fmaplen; m++)
+    {
+        printf("map[%d] = %lld\n", m, map[m]);
+    }
+
+    free(map);
+    free(gdims);
+        
+    /* Free the PIO decomposition. */
+    printf("%d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+        
+    return 0;
+}
+
+/** 
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int ioid2;             /* ID for decomposition we will create from file. */
+    char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+    char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int num_decomp_file_types = 1;
+    int ret;              /* Return code. */
+
+#ifdef _NETCDF4
+    /* Two extra output methods to tests if NetCDF-4 is present. */
+    num_decomp_file_types = 3;
+#endif /* _NETCDF4 */
+    
+    for (int decomp_file_type = 0; decomp_file_type < num_decomp_file_types; decomp_file_type++)
+    {
+        int cmode = 0;
+            
+        /* Determine the create mode. */
+        if (decomp_file_type)
+            cmode |= NC_NETCDF4;
+        if (decomp_file_type == 2)
+            cmode |= NC_MPIIO;
+            
+        /* Use PIO to create the decomp file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++) 
+        {
+            /* Create the filename. */
+            sprintf(filename, "decomp_%s_iotype_%d_deomp_type_%d.nc", TEST_NAME, flavor[fmt],
+                    decomp_file_type);
+
+            printf("writing decomp file %s\n", filename);
+            if ((ret = PIOc_write_nc_decomp(iosysid, filename, cmode, ioid, test_comm, NULL,
+                                            NULL, 0)))
+                return ret;
+    
+            /* Read the data. */
+            printf("reading decomp file %s\n", filename);
+            if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT,
+                                           title_in, history_in, &fortran_order_in)))
+                return ret;
+    
+            /* Check the results. */
+            {
+                iosystem_desc_t *ios;
+                io_desc_t *iodesc;
+            
+                /* Get the IO system info. */
+                if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                    return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+                /* Get the IO desc, which describes the decomposition. */
+                if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                    return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+                if (iodesc->ioid != ioid2 || iodesc->maplen != TARGET_NTASKS || iodesc->ndims != NDIM2 ||
+                    iodesc->nrecvs != 1 || iodesc->ndof != TARGET_NTASKS || iodesc->num_aiotasks != TARGET_NTASKS
+                    || iodesc->rearranger != PIO_REARR_SUBSET || iodesc->maxregions != 1 ||
+                    iodesc->needsfill || iodesc->basetype != MPI_INT)
+                    return ERR_WRONG;
+                for (int e = 0; e < iodesc->maplen; e++)
+                    if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1)
+                        return ERR_WRONG;
+                if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN)
+                    return ERR_WRONG;
+                printf("%d in my test iodesc->maxiobuflen = %d\n", my_rank, iodesc->maxiobuflen);
+            }
+        
+
+            /* Free the PIO decomposition. */
+            if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+                ERR(ret);
+        }
+    }
+    return PIO_NOERR;
+}
+
+/* Run decomp tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks;  /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    MPI_Group world_group;      /* An MPI group of world. */
+    MPI_Comm test_comm;
+    int num_flavors;            /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS];    /* iotypes for the supported netCDF IO flavors. */
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ioid;
+    int ret;                    /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Initialize PIO system on world. */
+        printf("%d about to call Init_Intracomm\n", my_rank);
+        if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, REARRANGER, &iosysid)))
+            ERR(ret);
+        printf("%d done with Init_Intracomm\n", my_rank);
+
+        /* Set the error handler. */
+        /*PIOc_Set_IOSystem_Error_Handling(iosysid, PIO_BCAST_ERROR);*/
+        printf("%d about to set iosystem error hanlder for world\n", my_rank);
+        if ((ret = PIOc_set_iosystem_error_handling(iosysid, PIO_BCAST_ERROR, NULL)))
+            ERR(ret);
+        printf("%d done setting iosystem error hanlder for world\n", my_rank);
+
+        /* Get MPI_Group of world comm. */
+        if ((ret = MPI_Comm_group(test_comm, &world_group)))
+            ERR(ret);
+
+        /* Test basic decomp stuff. */
+        if ((ret = test_decomp1(iosysid, my_rank, test_comm)))
+            return ret;
+
+        /* Test PIOc_InitDecomp_bc(). */
+        if ((ret = test_decomp_bc(iosysid, my_rank, test_comm)))
+            return ret;
+
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, &ioid,
+                                           PIO_INT)))
+            return ret;
+
+        /* Test decomposition read/write. */
+        if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, test_comm)))
+            return ret;
+    
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+        
+        /* Finalize PIO systems. */
+        printf("%d pio finalized\n", my_rank);
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+        /* Free MPI resources used by test. */
+        if ((ret = MPI_Group_free(&world_group)))
+            ERR(ret);
+
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_intercomm2.c b/src/externals/pio2/tests/cunit/test_intercomm2.c
new file mode 100644
index 00000000000..0a6f4a5b2ff
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_intercomm2.c
@@ -0,0 +1,559 @@
+/*
+ * Tests for PIOc_Intercomm. This tests the Init_Intercomm()
+ * function, and basic asynch I/O capability.
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_intercomm2
    
+ *
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_intercomm2"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 2
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the test data. */
+#define NDIM 1
+
+/* The length of our test data. */
+#define DIM_LEN 4
+
+/* Number of netCDF atomic types. */
+#define NUM_TYPES 11
+
+/* The name of the dimension in the netCDF output file. */
+#define FIRST_DIM_NAME "jojo"
+#define DIM_NAME "dim_test_intercomm2"
+
+/* The name of the variable in the netCDF output file. */
+#define FIRST_VAR_NAME "bill"
+#define VAR_NAME "var_test_intercomm2"
+
+/* The name of the global attribute in the netCDF output file. */
+#define FIRST_ATT_NAME "willy_gatt_test_intercomm2"
+#define ATT_NAME "gatt_test_intercomm2"
+#define SHORT_ATT_NAME "short_gatt_test_intercomm2"
+#define FLOAT_ATT_NAME "float_gatt_test_intercomm2"
+#define DOUBLE_ATT_NAME "double_gatt_test_intercomm2"
+
+/* The value of the global attribute in the netCDF output file. */
+#define ATT_VALUE 42
+
+/* Check the file for correctness. */
+int check_file(int iosysid, int format, char *filename, int my_rank)
+{
+    int ncid;
+    int ret;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    int dimid2;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+    char dimname2[NC_MAX_NAME + 1];
+    PIO_Offset dimlen2;
+    char varname[NC_MAX_NAME + 1];
+    nc_type vartype;
+    int varndims, vardimids, varnatts;
+    char varname2[NC_MAX_NAME + 1];
+    nc_type vartype2;
+    int varndims2, vardimids2, varnatts2;
+    int varid2;
+    int att_data;
+    short short_att_data;
+    float float_att_data;
+    double double_att_data;
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+
+    /* Create a name that is too long. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    /* Re-open the file to check it. */
+    printf("%d test_intercomm2 opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename,
+                             NC_NOWRITE)))
+        ERR(ret);
+
+    /* Try to read the data. */
+    PIO_Offset start[NDIM] = {0}, count[NDIM] = {DIM_LEN};
+    int data_in[DIM_LEN];
+    /* Check with various combinations of start/count */
+    for (int start_index = 0; start_index < DIM_LEN; start_index++)
+    {
+        start[0] = start_index;
+        count[0] = DIM_LEN - start_index;
+        if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in)))
+            ERR(ret);
+        for (int i = 0; i < count[0]; i++)
+        {
+            printf("%d test_intercomm2 read data_in[%d] = %d, start_index = %d\n", my_rank, i, data_in[i], start_index);
+            if (data_in[i] != (i + start_index))
+                ERR(ERR_AWFUL);
+        }
+    }
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        ERR(ret);
+    if (ndims != 1 || nvars != 1 || ngatts != 4 || unlimdimid != -1)
+        ERR(ERR_WRONG);
+
+    /* This should return PIO_NOERR. */
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        ERR(ret);
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        ERR(ret);
+    if (ndims2 != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        ERR(ret);
+    if (nvars2 != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        ERR(ret);
+    if (ngatts2 != 4)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        ERR(ret);
+    if (unlimdimid != -1)
+        ERR(ERR_WRONG);
+    /* Should succeed, do nothing. */
+    if ((ret = PIOc_inq_unlimdim(ncid, NULL)))
+        ERR(ret);
+
+    /* Check out the dimension. */
+    if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen)))
+        ERR(ret);
+    if (strcmp(dimname, DIM_NAME) || dimlen != DIM_LEN)
+        ERR(ERR_WRONG);
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+        ERR(ret);
+    if (strcmp(dimname2, DIM_NAME))
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2)))
+        ERR(ret);
+    if (dimlen2 != DIM_LEN)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2)))
+        ERR(ret);
+    if (dimid2 != 0)
+        ERR(ERR_WRONG);
+
+    /* These should not work. */
+    if (PIOc_inq_dimid(ncid + TEST_VAL_42, DIM_NAME, &dimid2) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_dimid(ncid, too_long_name, &dimid2) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Check out the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts)))
+        ERR(ret);
+    if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM ||
+        vardimids != 0 || varnatts != 0)
+        ERR(ERR_WRONG);
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+        ERR(ret);
+    if (strcmp(varname2, VAR_NAME))
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2)))
+        ERR(ret);
+    if (vartype2 != NC_INT)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2)))
+        ERR(ret);
+    if (varndims2 != NDIM)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2)))
+        ERR(ret);
+    if (vardimids2 != 0)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2)))
+        ERR(ret);
+    if (varnatts2 != 0)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2)))
+        ERR(ret);
+    if (varid2 != 0)
+        ERR(ERR_WRONG);
+
+    /* These should not work. */
+    if (PIOc_inq_varid(ncid + TEST_VAL_42, VAR_NAME, &varid2) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_varid(ncid, NULL, &varid2) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_varid(ncid, too_long_name, &varid2) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Check out the global attributes. */
+    nc_type atttype;
+    PIO_Offset attlen;
+    char myattname[NC_MAX_NAME + 1];
+    int myid;
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen)))
+        ERR(ret);
+    if (atttype != NC_INT || attlen != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen)))
+        ERR(ret);
+    if (attlen != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, myattname)))
+        ERR(ret);
+    if (strcmp(ATT_NAME, myattname))
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_attid(ncid, NC_GLOBAL, ATT_NAME, &myid)))
+        ERR(ret);
+    if (PIOc_inq_attid(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &myid) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (myid != 0)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data)))
+        ERR(ret);
+    printf("%d test_intercomm2 att_data = %d\n", my_rank, att_data);
+    if (att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, SHORT_ATT_NAME, &atttype, &attlen)))
+        ERR(ret);
+    if (atttype != NC_SHORT || attlen != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, &short_att_data)))
+        ERR(ret);
+    if (short_att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, &float_att_data)))
+        ERR(ret);
+    if (float_att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, &double_att_data)))
+        ERR(ret);
+    if (double_att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+
+    /* These should not work. */
+    if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &atttype, &attlen) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, NULL, &atttype, &attlen) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, too_long_name, &atttype, &attlen) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_get_att(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &att_data) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_get_att(ncid, NC_GLOBAL, NULL, &att_data) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+    if (PIOc_get_att(ncid, NC_GLOBAL, too_long_name, &att_data) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Close the file. */
+    printf("%d test_intercomm2 closing file (again) ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_closefile(ncid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Run Tests for Init_Intercomm. */
+int main(int argc, char **argv)
+{
+    /* Zero-based rank of processor. */
+    int my_rank;
+
+    /* Number of processors involved in current execution. */
+    int ntasks;
+
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+
+    /* Names for the output files. */
+    char filename[NUM_FLAVORS][NC_MAX_NAME + 1];
+
+    /* The ID for the parallel I/O system. */
+    int iosysid[COMPONENT_COUNT];
+
+    /* Return code. */
+    int ret;
+
+    MPI_Comm test_comm;
+
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+
+    /* Create a name that is too long. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+    
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Figure out iotypes. */
+    if ((ret = get_iotypes(&num_flavors, flavor)))
+        ERR(ret);
+
+    if(my_rank < TARGET_NTASKS)
+    {
+	printf("%d: test_intercomm2 ParallelIO Library test_intercomm2 running on %d processors.\n",
+	       my_rank, ntasks);
+
+        /* Initialize the PIO IO system. This specifies how many and which
+         * processors are involved in I/O. */
+
+        /* Turn on logging. */
+        if ((ret = PIOc_set_log_level(3)))
+            ERR(ret);
+
+        /* How many processors will be used for our IO and 2 computation components. */
+        int num_procs[COMPONENT_COUNT + 1] = {2, 2};
+
+        /* Is the current process a computation task? */
+        int comp_task = my_rank < 2 ? 0 : 1;
+
+        /* Index of computation task in iosysid array. Varies by rank and
+         * does not apply to IO component processes. */
+        int my_comp_idx = comp_task ? 0 : -1;
+
+        /* Initialize the IO system. */
+        if ((ret = PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                                   num_procs, NULL, NULL, NULL, iosysid)))
+            ERR(ERR_AWFUL);
+
+	printf("%d: test_intercomm2 ParallelIO Library test_intercomm2 comp task returned.\n",
+	       my_rank);
+
+        /* All the netCDF calls are only executed on the computation
+         * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+         * and when the do, they should go straight to finalize. */
+        if (comp_task)
+        {
+            for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                int ncid, varid, dimid;
+                PIO_Offset start[NDIM], count[NDIM] = {0};
+                int data[DIM_LEN];
+
+                /* Create the filename for this flavor. */
+                sprintf(filename[fmt], "test_intercomm2_%d.nc", flavor[fmt]);
+
+                /* Create a netCDF file with one dimension and one variable. */
+		printf("%d test_intercomm2 creating file %s\n", my_rank, filename[fmt]);
+                if ((ret = PIOc_createfile(iosysid[my_comp_idx], &ncid, &flavor[fmt], filename[fmt],
+                                           NC_CLOBBER)))
+                    ERR(ret);
+		printf("%d test_intercomm2 file created ncid = %d\n", my_rank, ncid);
+
+                /* End define mode, then re-enter it. */
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+		printf("%d test_intercomm2 calling redef\n", my_rank);
+                if ((ret = PIOc_redef(ncid)))
+                    ERR(ret);
+
+                /* Test the inq_format function. */
+                int myformat;
+                if (PIOc_inq_format(ncid + TEST_VAL_42, &myformat) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_inq_format(ncid, &myformat)))
+                    ERR(ret);
+                if ((flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF) &&
+                    myformat != 1)
+                    ERR(ERR_AWFUL);
+                else if ((flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) &&
+                         myformat != 3)
+                    ERR(ERR_AWFUL);
+
+                /* Test the inq_type function for atomic types. */
+                char type_name[NC_MAX_NAME + 1];
+                PIO_Offset type_size;
+                nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
+                                            NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
+                int type_len[NUM_TYPES] = {1, 1, 2, 4, 4, 8, 1, 2, 4, 8, 8};
+                int max_type = flavor[fmt] == PIO_IOTYPE_NETCDF ? NC_DOUBLE : NC_UINT64;
+
+                /* This should not work. */
+                if (PIOc_inq_type(ncid + TEST_VAL_42, xtype[0], type_name, &type_size) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+
+                /* These should work. */
+                for (int i = 0; i < max_type; i++)
+                {
+                    if ((ret = PIOc_inq_type(ncid, xtype[i], type_name, &type_size)))
+                        ERR(ret);
+                    if (type_size != type_len[i])
+                        ERR(ERR_AWFUL);
+                }
+
+                /* Define a dimension. */
+                char dimname2[NC_MAX_NAME + 1];
+		printf("%d test_intercomm2 defining dimension %s\n", my_rank, DIM_NAME);
+                if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME, DIM_LEN, &dimid)))
+                    ERR(ret);
+                if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+                    ERR(ret);
+                if (strcmp(dimname2, FIRST_DIM_NAME))
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_rename_dim(ncid, 0, DIM_NAME)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_rename_dim(ncid + TEST_VAL_42, 0, DIM_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_dim(ncid, 0, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_dim(ncid, 0, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+
+                /* Define a 1-D variable. */
+                char varname2[NC_MAX_NAME + 1];
+		printf("%d test_intercomm2 defining variable %s\n", my_rank, VAR_NAME);
+                if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME, NC_INT, NDIM, &dimid, &varid)))
+                    ERR(ret);
+                if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+                    ERR(ret);
+                if (strcmp(varname2, FIRST_VAR_NAME))
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_rename_var(ncid, 0, VAR_NAME)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_rename_var(ncid + TEST_VAL_42, 0, VAR_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_var(ncid, 0, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_var(ncid, 0, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+
+                /* Add a global attribute. */
+		printf("%d test_intercomm2 writing attributes %s\n", my_rank, ATT_NAME);
+                int att_data = ATT_VALUE;
+                short short_att_data = ATT_VALUE;
+                float float_att_data = ATT_VALUE;
+                double double_att_data = ATT_VALUE;
+                char attname2[NC_MAX_NAME + 1];
+
+                /* Write an att and rename it. */
+                if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2)))
+                    ERR(ret);
+                if (strcmp(attname2, FIRST_ATT_NAME))
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_inq_attname(ncid + TEST_VAL_42, NC_GLOBAL, 0, attname2) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_att(ncid + TEST_VAL_42, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_del_att(ncid + TEST_VAL_42, NC_GLOBAL, FIRST_ATT_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_del_att(ncid, NC_GLOBAL, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_del_att(ncid, NC_GLOBAL, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+
+                /* Write an att and delete it. */
+                if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME)))
+                    ERR(ret);
+                /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL, NULL)) != PIO_ENOTATT) */
+                /* { */
+                /*      printf("ret = %d\n", ret); */
+                /*      ERR(ERR_AWFUL); */
+                /* } */
+
+                /* Write some atts of different types. */
+                if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_put_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, NC_DOUBLE, 1, &double_att_data)))
+                    ERR(ret);
+
+                /* Check some att types. */
+                nc_type myatttype;
+                if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, SHORT_ATT_NAME, &myatttype)))
+                    ERR(ret);
+                if (myatttype != NC_SHORT)
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, FLOAT_ATT_NAME, &myatttype)))
+                    ERR(ret);
+                if (myatttype != NC_FLOAT)
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, &myatttype)))
+                    ERR(ret);
+                if (myatttype != NC_DOUBLE)
+                    ERR(ERR_WRONG);
+
+                /* End define mode. */
+		printf("%d test_intercomm2 ending define mode ncid = %d\n", my_rank, ncid);
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+
+                /* Write some data. For the PIOc_put/get functions, all
+                 * data must be on compmaster before the function is
+                 * called. Only compmaster's arguments are passed to the
+                 * async msg handler. All other computation tasks are
+                 * ignored. */
+                for (int i = 0; i < DIM_LEN; i++)
+                    data[i] = i;
+		printf("%d test_intercomm2 writing data\n", my_rank);
+		printf("%d test_intercomm2 writing data\n", my_rank);
+                start[0] = 0;
+                count[0] = DIM_LEN;
+                if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
+                    ERR(ret);
+
+                /* Close the file. */
+		printf("%d test_intercomm2 closing file ncid = %d\n", my_rank, ncid);
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* Check the file for correctness. */
+                if ((ret = check_file(iosysid[my_comp_idx], flavor[fmt], filename[fmt], my_rank)))
+                    ERR(ret);
+
+                /* Now delete the file. */
+                /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */
+                /*      ERR(ret); */
+                /* if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename[fmt], */
+                /*                           NC_NOWRITE)) != PIO_ENFILE) */
+                /*      ERR(ERR_AWFUL); */
+
+            } /* next netcdf flavor */
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+	    printf("%d test_intercomm2 Freeing PIO resources\n", my_rank);
+            if ((ret = PIOc_finalize(iosysid[my_comp_idx])))
+                ERR(ret);
+        }
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem2.c b/src/externals/pio2/tests/cunit/test_iosystem2.c
new file mode 100644
index 00000000000..ca27acad514
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem2.c
@@ -0,0 +1,226 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem2"
+
+/* Used to define netcdf test file. */
+#define PIO_TF_MAX_STR_LEN 100
+#define ATTNAME "filename"
+#define DIMNAME "filename_dim"
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int create_file(MPI_Comm comm, int iosysid, int format, char *filename,
+                char *attname, char *dimname, int my_rank)
+{
+    int ncid, varid, dimid;
+    int ret;
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* Use the ncid to set the IO system error handler. This function
+     * is deprecated. */
+    PIOc_Set_File_Error_Handling(ncid, PIO_RETURN_ERROR);
+    int method = PIOc_Set_File_Error_Handling(ncid, PIO_RETURN_ERROR);
+    if (method != PIO_RETURN_ERROR)
+        return ERR_WRONG;
+
+    /* Define a dimension. */
+    printf("%d defining dimension %s\n", my_rank, dimname);
+    if ((ret = PIOc_def_dim(ncid, dimname, PIO_TF_MAX_STR_LEN, &dimid)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    printf("%d defining variable %s\n", my_rank, attname);
+    if ((ret = PIOc_def_var(ncid, attname, NC_CHAR, 1, &dimid, &varid)))
+        return ret;
+
+    /* Write an attribute. */
+    if ((ret = PIOc_put_att_text(ncid, varid, attname, strlen(filename), filename)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Close the file. */
+    printf("%d closing file ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+    printf("%d closed file ncid = %d\n", my_rank, ncid);
+
+    return PIO_NOERR;
+}
+
+/* This checks an already-open netCDF file. */
+int check_file(MPI_Comm comm, int iosysid, int format, int ncid, char *filename,
+               char *attname, char *dimname, int my_rank)
+{
+    int dimid;
+    int ret;
+
+    /* Check the file. */
+    if ((ret = PIOc_inq_dimid(ncid, dimname, &dimid)))
+        return ret;
+    printf("%d dimid = %d\n", my_rank, dimid);
+
+    return PIO_NOERR;
+}
+
+/* This opens and checks a netCDF file. */
+int open_and_check_file(MPI_Comm comm, int iosysid, int iotype, int *ncid, char *fname,
+                        char *attname, char *dimname, int disable_close, int my_rank)
+{
+    int mode = PIO_WRITE;
+    int ret;
+
+    /* Open the file. */
+    if ((ret = PIOc_openfile(iosysid, ncid, &iotype, fname, mode)))
+        return ret;
+
+    /* Check the file. */
+    if ((ret = check_file(comm, iosysid, iotype, *ncid, fname, attname, dimname, my_rank)))
+        return ret;
+
+    /* Close the file, maybe. */
+    if (!disable_close)
+        if ((ret = PIOc_closefile(*ncid)))
+            return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int ret; /* Return code. */
+    int num_flavors;
+    int iotypes[NUM_FLAVORS];
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, iotypes)))
+            ERR(ret);
+
+        /* Split world into odd and even. */
+        MPI_Comm newcomm;
+        int even = my_rank % 2 ? 0 : 1;
+        if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
+            MPIERR(ret);
+        printf("%d newcomm = %d even = %d\n", my_rank, newcomm, even);
+
+        /* Get rank in new communicator and its size. */
+        int new_rank, new_size;
+        if ((ret = MPI_Comm_rank(newcomm, &new_rank)))
+            MPIERR(ret);
+        if ((ret = MPI_Comm_size(newcomm, &new_size)))
+            MPIERR(ret);
+        printf("%d newcomm = %d new_rank = %d new_size = %d\n", my_rank, newcomm,
+               new_rank, new_size);
+
+        /* Initialize PIO system. */
+        if ((ret = PIOc_Init_Intracomm(newcomm, 2, 1, 0, 1, &iosysid)))
+            ERR(ret);
+
+        /* This should fail. */
+        if (PIOc_finalize(iosysid + TEST_VAL_42) != PIO_EBADID)
+            ERR(ERR_WRONG);
+
+        /* Initialize another PIO system. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, 4, 1, 0, 1, &iosysid_world)))
+            ERR(ret);
+
+        for (int i = 0; i < num_flavors; i++)
+        {
+            char fname0[] = "pio_iosys_test_file0.nc";
+            char fname1[] = "pio_iosys_test_file1.nc";
+            char fname2[] = "pio_iosys_test_file2.nc";
+            printf("\n\n%d i = %d\n", my_rank, i);
+
+            if ((ret = create_file(test_comm, iosysid_world, iotypes[i], fname0, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, iotypes[i], fname1, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, iotypes[i], fname2, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            MPI_Barrier(test_comm);
+
+            /* Now check the first file. */
+            int ncid;
+            if ((ret = open_and_check_file(test_comm, iosysid_world, iotypes[i], &ncid, fname0,
+                                           ATTNAME, DIMNAME, 1, my_rank)))
+                ERR(ret);
+
+            /* Now have the odd/even communicators each check one of the
+             * remaining files. */
+            int ncid2;
+            char *fname = even ? fname1 : fname2;
+            printf("\n***\n");
+            if ((ret = open_and_check_file(newcomm, iosysid, iotypes[i], &ncid2, fname,
+                                           ATTNAME, DIMNAME, 1, my_rank)))
+                ERR(ret);
+
+
+            /* Close the still-open files. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        } /* next iotype */
+        if ((ret = MPI_Comm_free(&newcomm)))
+            MPIERR(ret);
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem2_simple.c b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c
new file mode 100644
index 00000000000..4cf217e72e5
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c
@@ -0,0 +1,233 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 2
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem2_simple"
+
+/* Number of test files generated. */
+#define NUM_FILES 2
+
+/* Used to define netcdf test file. */
+#define DIMLEN 1
+
+/* Needed to init intracomm. */
+#define STRIDE 1
+#define BASE 0
+#define REARRANGER 1
+
+/* Ten megabytes. */
+#define TEN_MEG 10485760
+
+/* Run test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    PIO_Offset oldlimit;
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if(my_rank < TARGET_NTASKS)
+    {
+        /* Try setting the blocksize. */
+        if ((ret = PIOc_set_blocksize(2048)))
+            ERR(ret);
+
+        /* Try setting the buffer size limit. */
+        oldlimit = PIOc_set_buffer_size_limit(200000);
+        if (oldlimit != TEN_MEG)
+            ERR(ERR_WRONG);
+        oldlimit = PIOc_set_buffer_size_limit(TEN_MEG);
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Split world into odd and even. */
+        MPI_Comm newcomm;
+        int even = my_rank % 2 ? 0 : 1;
+        if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
+            MPIERR(ret);
+
+        /* Get size of new communicator. */
+        int new_size;
+        if ((ret = MPI_Comm_size(newcomm, &new_size)))
+            MPIERR(ret);
+        printf("%d newcomm = %d even = %d new_size = %d\n", my_rank, newcomm, even, new_size);
+
+        /* Check that some bad inputs are rejected. */
+        if (PIOc_Init_Intracomm(newcomm, new_size, STRIDE + 30, BASE, REARRANGER,
+                                &iosysid) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_Init_Intracomm(newcomm, new_size, STRIDE, BASE, REARRANGER, NULL) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Initialize an intracomm for evens/odds. */
+        if ((ret = PIOc_Init_Intracomm(newcomm, new_size, STRIDE, BASE, REARRANGER, &iosysid)))
+            ERR(ret);
+
+        /* Test some support functions. */
+        bool active;
+        if ((ret = PIOc_iosystem_is_active(iosysid, &active)))
+            ERR(ret);
+        if (!active)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_iosystem_is_active(iosysid + 1, &active)))
+            ERR(ret);
+        if (active)
+            ERR(ERR_WRONG);
+
+        int numiotasks;
+        if (PIOc_get_numiotasks(iosysid + TEST_VAL_42, &numiotasks) != PIO_EBADID)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_get_numiotasks(iosysid, NULL)))
+            ERR(ret);
+        if ((ret = PIOc_get_numiotasks(iosysid, &numiotasks)))
+            ERR(ret);
+        printf("%d numiotasks = %d\n", my_rank, numiotasks);
+        if (numiotasks != 1)
+            ERR(ERR_WRONG);
+
+        int iorank;
+        if (PIOc_iotask_rank(iosysid + TEST_VAL_42, &iorank) != PIO_EBADID)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_iotask_rank(iosysid, NULL)))
+            ERR(ret);
+        if ((ret = PIOc_iotask_rank(iosysid, &iorank)))
+            ERR(ret);
+        printf("%d iorank = %d\n", my_rank, iorank);
+        /* Each of two tasks has an iosystem. On both iosystems, the
+         * single task has iorank of zero. */
+        if (iorank != 0)
+            ERR(ERR_WRONG);
+
+        /* Both tasks are IO tasks. */
+        bool ioproc;
+        if (PIOc_iam_iotask(iosysid + TEST_VAL_42, &ioproc) != PIO_EBADID)
+            ERR(ret);
+        if ((ret = PIOc_iam_iotask(iosysid, NULL)))
+            ERR(ret);
+        if ((ret = PIOc_iam_iotask(iosysid, &ioproc)))
+            ERR(ret);
+        if (!ioproc)
+            ERR(ERR_WRONG);
+
+        /* Initialize an intracomm for all processes. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, STRIDE, BASE, REARRANGER,
+                                       &iosysid_world)))
+            ERR(ret);
+
+        int ncid;
+        int ncid2;
+        for (int i = 0; i < num_flavors; i++)
+        {
+            char fn[NUM_FILES][NC_MAX_NAME + 1];
+            char dimname[NUM_FILES][NC_MAX_NAME + 1];
+
+            /* Create the test files. */
+            for (int f = 0; f < NUM_FILES; f++)
+            {
+                int lncid, dimid;
+
+                /* Creat file and dimension name. */
+                sprintf(fn[f], "pio_iosys_test_file%d.nc", f);
+                sprintf(dimname[f], "dim_%d", f);
+
+                if ((ret = PIOc_createfile(iosysid_world, &lncid, &flavor[i], fn[f], NC_CLOBBER)))
+                    return ret;
+                /* Define a dimension. */
+                if ((ret = PIOc_def_dim(lncid, dimname[f], DIMLEN, &dimid)))
+                    return ret;
+                if ((ret = PIOc_enddef(lncid)))
+                    return ret;
+                if ((ret = PIOc_closefile(lncid)))
+                    return ret;
+            }
+
+            /* These should not work. */
+            if (PIOc_openfile(iosysid_world + TEST_VAL_42, &ncid, &flavor[i], fn[0], PIO_WRITE) != PIO_EBADID)
+                return ERR_WRONG;
+            if (PIOc_openfile(iosysid_world, NULL, &flavor[i], fn[0], PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+            if (PIOc_openfile(iosysid_world, &ncid, NULL, fn[0], PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+            if (PIOc_openfile(iosysid_world, &ncid, &flavor[i], NULL, PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+            int bad_iotype = flavor[i] + TEST_VAL_42;
+            if (PIOc_openfile(iosysid_world, &ncid, &bad_iotype, fn[0], PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+
+            /* Open the first file with world iosystem. */
+            if ((ret = PIOc_openfile(iosysid_world, &ncid, &flavor[i], fn[0], PIO_WRITE)))
+                return ret;
+
+            /* Check the first file. */
+            char dimname_in[NC_MAX_NAME + 1];
+            if ((ret = PIOc_inq_dimname(ncid, 0, dimname_in)))
+                return ret;
+            printf("%d ncid dimname_in = %s should be %s\n", my_rank, dimname_in, dimname[0]);
+            if (strcmp(dimname_in, dimname[0]))
+                return ERR_WRONG;
+
+            /* Check the other file with the other IO. */
+            if (even)
+            {
+                if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[i], fn[1], PIO_WRITE)))
+                    return ret;
+                if ((ret = PIOc_inq_dimname(ncid2, 0, dimname_in)))
+                    return ret;
+                printf("%d ncid2 dimname_in = %s should be %s\n", my_rank, dimname_in, dimname[1]);
+                if (strcmp(dimname_in, dimname[1]))
+                    return ERR_WRONG;
+            }
+
+            /* Close the still-open files. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+            if (even)
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+        } /* next iotype */
+        if ((ret = MPI_Comm_free(&newcomm)))
+            MPIERR(ret);
+
+        /* Finalize PIO odd/even intracomm. */
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+
+        /* Finalize PIO world intracomm. */
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    }/* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem2_simple2.c b/src/externals/pio2/tests/cunit/test_iosystem2_simple2.c
new file mode 100644
index 00000000000..018caaedb9d
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem2_simple2.c
@@ -0,0 +1,142 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem2_simple2"
+
+/* Number of test files generated. */
+#define NUM_FILES 3
+
+/* Used to define netcdf test file. */
+#define PIO_TF_MAX_STR_LEN 100
+#define ATTNAME "filename"
+#define DIMNAME "filename_dim"
+
+/* Needed to init intracomm. */
+#define STRIDE 1
+#define BASE 0
+#define REARRANGER 1
+
+/* Run test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Only do something on the first TARGET_NTASKS tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+	/* Figure out iotypes. */
+	if ((ret = get_iotypes(&num_flavors, flavor)))
+	    ERR(ret);
+
+        /* Split world into odd and even. */
+        MPI_Comm newcomm;
+        int even = my_rank % 2 ? 0 : 1;
+        if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
+            MPIERR(ret);
+        printf("%d newcomm = %d even = %d\n", my_rank, newcomm, even);
+
+        /* Get size of new communicator. */
+        int new_size;
+        if ((ret = MPI_Comm_size(newcomm, &new_size)))
+            MPIERR(ret);
+
+        /* Initialize an intracomm for evens/odds. */
+        if ((ret = PIOc_Init_Intracomm(newcomm, new_size, STRIDE, BASE, REARRANGER, &iosysid)))
+            ERR(ret);
+
+        /* Initialize an intracomm for all processes. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, STRIDE, BASE, REARRANGER,
+                                       &iosysid_world)))
+            ERR(ret);
+
+        for (int flv = 0; flv < num_flavors; flv++)
+        {
+            char filename[NUM_SAMPLES][NC_MAX_NAME + 1]; /* Test filename. */
+            int sample_ncid[NUM_SAMPLES];
+
+            for (int sample = 0; sample < NUM_SAMPLES; sample++)
+            {
+                char iotype_name[NC_MAX_NAME + 1];
+
+                /* Create a filename. */
+                if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                    return ret;
+                sprintf(filename[sample], "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, 0);
+
+                /* Create sample file. */
+                printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename[sample]);
+                if ((ret = create_nc_sample(sample, iosysid_world, flavor[flv], filename[sample],
+					    my_rank, NULL)))
+                    ERR(ret);
+
+                /* Check the file for correctness. */
+                if ((ret = check_nc_sample(sample, iosysid_world, flavor[flv], filename[sample],
+					   my_rank, &sample_ncid[sample])))
+                    ERR(ret);
+
+            }
+
+            /* Now check the files with the other iosysid. Even and odd
+             * processes will check different files. */
+            int this_sample = even ? 0 : 1;
+            char *file1 = filename[this_sample];
+            int ncid2;
+            if ((ret = check_nc_sample(this_sample, iosysid, flavor[flv], file1, my_rank, &ncid2)))
+                ERR(ret);
+
+            /* Now close the open files. */
+            for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                if ((ret = PIOc_closefile(sample_ncid[sample])))
+                    ERR(ret);
+
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        } /* next iotype */
+        if ((ret = MPI_Comm_free(&newcomm)))
+            MPIERR(ret);
+
+        /* Finalize PIO odd/even intracomm. */
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+        /* Finalize PIO world intracomm. */
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    }  /* my_rank < TARGET_NTASKS */
+
+    /* Wait for all task before finalizing. */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem3.c b/src/externals/pio2/tests/cunit/test_iosystem3.c
new file mode 100644
index 00000000000..53b0c63b67c
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem3.c
@@ -0,0 +1,375 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran
+ * pio_iosystem_tests3.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem3"
+
+/* Used to define netcdf test file. */
+#define PIO_TF_MAX_STR_LEN 100
+#define ATTNAME "filename"
+#define DIMNAME "filename_dim"
+
+/* Used to devide up the tasks into MPI groups. */
+#define OVERLAP_NUM_RANGES 2
+#define EVEN_NUM_RANGES 1
+
+/* Used when initializing PIO. */
+#define STRIDE1 1
+#define STRIDE2 2
+#define BASE0 0
+#define BASE1 1
+#define NUM_IO1 1
+#define NUM_IO2 2
+#define NUM_IO4 4
+#define REARRANGER 2
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int create_file(MPI_Comm comm, int iosysid, int format, char *filename,
+                char *attname, char *dimname, int my_rank)
+{
+    int ncid, varid, dimid;
+    int ret;
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* Define a dimension. */
+    printf("%d defining dimension %s\n", my_rank, dimname);
+    if ((ret = PIOc_def_dim(ncid, dimname, PIO_TF_MAX_STR_LEN, &dimid)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    printf("%d defining variable %s\n", my_rank, attname);
+    if ((ret = PIOc_def_var(ncid, attname, NC_CHAR, 1, &dimid, &varid)))
+        return ret;
+
+    /* Write an attribute. */
+    if ((ret = PIOc_put_att_text(ncid, varid, attname, strlen(filename), filename)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Close the file. */
+    printf("%d closing file ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+    printf("%d closed file ncid = %d\n", my_rank, ncid);
+
+    return PIO_NOERR;
+}
+
+/* This checks an already-open netCDF file. */
+int check_file(MPI_Comm comm, int iosysid, int format, int ncid, char *filename,
+               char *attname, char *dimname, int my_rank)
+{
+    int dimid;
+    int varid;
+    char *att_data;
+    int ret;
+
+    /* Check the dimid. */
+    if ((ret = PIOc_inq_dimid(ncid, dimname, &dimid)))
+        return ret;
+    if (dimid)
+        return ERR_WRONG;
+
+    /* Check the varid (it's got the same name as the att). */
+    if ((ret = PIOc_inq_varid(ncid, attname, &varid)))
+        return ret;
+    if (varid)
+        return ERR_WRONG;
+
+    /* Check the attribute. Null terminating byte deliberately ignored
+     * to match fortran code. */
+    if (!(att_data = malloc(strlen(filename) * sizeof(char))))
+        return PIO_ENOMEM;
+    if ((ret = PIOc_get_att(ncid, varid, attname, att_data)))
+        return ret;
+    printf("%d DONE with get_att!!!\n", my_rank);
+    if (strncmp(att_data, filename, strlen(filename)))
+        return ERR_WRONG;
+    free(att_data);
+    printf("%d DONE with get_att!!!\n", my_rank);
+
+    return PIO_NOERR;
+}
+
+/* This opens and checks a netCDF file. */
+int open_and_check_file(MPI_Comm comm, int iosysid, int iotype, int *ncid, char *fname,
+                        char *attname, char *dimname, int disable_close, int my_rank)
+{
+    int mode = PIO_WRITE;
+    int ret;
+
+    /* Open the file. */
+    if ((ret = PIOc_openfile(iosysid, ncid, &iotype, fname, mode)))
+        return ret;
+
+    /* Check the file. */
+    if ((ret = check_file(comm, iosysid, iotype, *ncid, fname, attname, dimname, my_rank)))
+        return ret;
+
+    /* Close the file, maybe. */
+    if (!disable_close)
+        if ((ret = PIOc_closefile(*ncid)))
+            return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int even_iosysid; /* The ID for iosystem of even_comm. */
+    int overlap_iosysid; /* The ID for iosystem of even_comm. */
+    MPI_Group world_group; /* An MPI group of world. */
+    MPI_Group even_group; /* An MPI group of 0 and 2. */
+    MPI_Group overlap_group; /* An MPI group of 0, 1, and 3. */
+    MPI_Comm even_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 2 */
+    MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */
+    int even_rank = -1, overlap_rank = -1; /* Tasks rank in communicator. */
+    int even_size = 0, overlap_size = 0; /* Size of communicator. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Initialize PIO system on world. */
+        printf("%d about to call Init_Intracomm\n", my_rank);
+        if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, REARRANGER, &iosysid_world)))
+            ERR(ret);
+        printf("%d done with Init_Intracomm\n", my_rank);
+
+        /* Set the error handler. */
+        /*PIOc_Set_IOSystem_Error_Handling(iosysid_world, PIO_BCAST_ERROR);*/
+        printf("%d about to set iosystem error hanlder for world\n", my_rank);
+        if ((ret = PIOc_set_iosystem_error_handling(iosysid_world, PIO_BCAST_ERROR, NULL)))
+            ERR(ret);
+        printf("%d done setting iosystem error hanlder for world\n", my_rank);
+
+        /* Get MPI_Group of world comm. */
+        if ((ret = MPI_Comm_group(test_comm, &world_group)))
+            ERR(ret);
+
+        /* Create a group with tasks 0 and 2. */
+        int even_ranges[EVEN_NUM_RANGES][3] = {{0, 2, 2}};
+        if ((ret = MPI_Group_range_incl(world_group, EVEN_NUM_RANGES, even_ranges, &even_group)))
+            ERR(ret);
+
+        /* Create a communicator from the even_group. */
+        if ((ret = MPI_Comm_create(test_comm, even_group, &even_comm)))
+            ERR(ret);
+
+        /* Learn my rank and the total number of processors in even group. */
+        if (even_comm != MPI_COMM_NULL)
+        {
+            if ((ret = MPI_Comm_rank(even_comm, &even_rank)))
+                MPIERR(ret);
+            if ((ret = MPI_Comm_size(even_comm, &even_size)))
+                MPIERR(ret);
+        }
+        printf("%d even_comm = %d even_rank = %d even_size = %d\n", my_rank,
+               even_comm, even_rank, even_size);
+
+        /* Create a group with tasks 0, 1, and 3. */
+        int overlap_ranges[OVERLAP_NUM_RANGES][3] = {{0, 0, 1}, {1, 3, 2}};
+        if ((ret = MPI_Group_range_incl(world_group, OVERLAP_NUM_RANGES,
+                                        overlap_ranges, &overlap_group)))
+            ERR(ret);
+
+        /* Create a communicator from the overlap_group. */
+        if ((ret = MPI_Comm_create(test_comm, overlap_group, &overlap_comm)))
+            ERR(ret);
+
+        /* Learn my rank and the total number of processors in overlap
+         * group. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = MPI_Comm_rank(overlap_comm, &overlap_rank)))
+                MPIERR(ret);
+            if ((ret = MPI_Comm_size(overlap_comm, &overlap_size)))
+                MPIERR(ret);
+        }
+        printf("%d overlap_comm = %d overlap_rank = %d overlap_size = %d\n", my_rank,
+               overlap_comm, overlap_rank, overlap_size);
+
+        /* Initialize PIO system for even. */
+        if (even_comm != MPI_COMM_NULL)
+        {
+            if ((ret = PIOc_Init_Intracomm(even_comm, NUM_IO1, STRIDE1, BASE1, REARRANGER, &even_iosysid)))
+                ERR(ret);
+
+            /* These should not work. */
+            if (PIOc_set_hint(even_iosysid + TEST_VAL_42, NULL, NULL) != PIO_EBADID)
+                ERR(ERR_WRONG);
+            if (PIOc_set_hint(even_iosysid, NULL, NULL) != PIO_EINVAL)
+                ERR(ERR_WRONG);
+
+            /* Set the hint (which will be ignored). */
+            if ((ret = PIOc_set_hint(even_iosysid, "hint", "hint_value")))
+                ERR(ret);
+
+            /* Set the error handler. */
+            /*PIOc_Set_IOSystem_Error_Handling(even_iosysid, PIO_BCAST_ERROR);*/
+            printf("%d about to set iosystem error hanlder for even\n", my_rank);
+            if ((ret = PIOc_set_iosystem_error_handling(even_iosysid, PIO_BCAST_ERROR, NULL)))
+                ERR(ret);
+            printf("%d done setting iosystem error hanlder for even\n", my_rank);
+        }
+
+        /* Initialize PIO system for overlap comm. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = PIOc_Init_Intracomm(overlap_comm, NUM_IO2, STRIDE1, BASE1, REARRANGER,
+                                           &overlap_iosysid)))
+                ERR(ret);
+
+            printf("%d about to set iosystem error hanlder for overlap\n", my_rank);
+            /* Set the error handler. */
+            /* if ((ret = PIOc_set_iosystem_error_handling(overlap_iosysid, PIO_BCAST_ERROR))) */
+            /*     ERR(ret); */
+            PIOc_Set_IOSystem_Error_Handling(overlap_iosysid, PIO_BCAST_ERROR);
+            printf("%d done setting iosystem error hanlder for overlap\n", my_rank);
+        }
+
+        for (int i = 0; i < num_flavors; i++)
+        {
+            char fname0[] = "pio_iosys_test_file0.nc";
+            char fname1[] = "pio_iosys_test_file1.nc";
+            char fname2[] = "pio_iosys_test_file2.nc";
+            printf("\n\n%d i = %d\n", my_rank, i);
+
+            if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname0, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname1, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname2, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            /* Now check the first file from WORLD communicator. */
+            int ncid;
+            if ((ret = open_and_check_file(test_comm, iosysid_world, flavor[i], &ncid, fname0,
+                                           ATTNAME, DIMNAME, 1, my_rank)))
+                ERR(ret);
+
+            /* Now have the even communicators check the files. */
+            int ncid2;
+            if (even_comm != MPI_COMM_NULL)
+            {
+                printf("\n***\n%d Checking file for even_comm\n", my_rank);
+                if ((ret = open_and_check_file(even_comm, even_iosysid, flavor[i], &ncid2, fname2,
+                                               ATTNAME, DIMNAME, 1, my_rank)))
+                    ERR(ret);
+                if ((ret = check_file(even_comm, even_iosysid, flavor[i], ncid2, fname2,
+                                      ATTNAME, DIMNAME, my_rank)))
+                    ERR(ret);
+            }
+
+            /* Now have the overlap communicators check the files. */
+            int ncid3;
+            if (overlap_comm != MPI_COMM_NULL)
+            {
+                printf("\n***%d Checking file for overlap_comm\n", my_rank);
+                if ((ret = open_and_check_file(overlap_comm, overlap_iosysid, flavor[i], &ncid3, fname1,
+                                               ATTNAME, DIMNAME, 1, my_rank)))
+                    ERR(ret);
+                if ((ret = check_file(overlap_comm, overlap_iosysid, flavor[i], ncid3, fname1,
+                                      ATTNAME, DIMNAME, my_rank)))
+                    ERR(ret);
+            }
+
+            /* Close the still-open files. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+            if (even_comm != MPI_COMM_NULL)
+            {
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+            }
+            if (overlap_comm != MPI_COMM_NULL)
+            {
+                if ((ret = PIOc_closefile(ncid3)))
+                    ERR(ret);
+            }
+        } /* next iotype */
+        /* Finalize PIO systems. */
+        printf("%d pio finalizing %d\n", my_rank, even_iosysid);
+        if (even_comm != MPI_COMM_NULL)
+            if ((ret = PIOc_finalize(even_iosysid)))
+                ERR(ret);
+        printf("%d pio finalizing %d\n", my_rank, overlap_iosysid);
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            printf("%d calling PIOc_finalize with iosysid = %d\n", my_rank, overlap_iosysid);
+            if ((ret = PIOc_finalize(overlap_iosysid)))
+                ERR(ret);
+        }
+        printf("%d pio finalized\n", my_rank);
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+
+        /* Free MPI resources used by test. */
+        if ((ret = MPI_Group_free(&overlap_group)))
+            ERR(ret);
+        if ((ret = MPI_Group_free(&even_group)))
+            ERR(ret);
+        if ((ret = MPI_Group_free(&world_group)))
+            ERR(ret);
+        if (overlap_comm != MPI_COMM_NULL)
+            if ((ret = MPI_Comm_free(&overlap_comm)))
+                ERR(ret);
+        if (even_comm != MPI_COMM_NULL)
+            if ((ret = MPI_Comm_free(&even_comm)))
+                ERR(ret);
+
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem3_simple.c b/src/externals/pio2/tests/cunit/test_iosystem3_simple.c
new file mode 100644
index 00000000000..7ac3ad2ebad
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem3_simple.c
@@ -0,0 +1,123 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem3_simple"
+
+/* Needed to init intracomm. */
+#define STRIDE 1
+#define BASE 0
+#define REARRANGER 1
+
+/* Used to devide up the tasks into MPI groups. */
+#define OVERLAP_NUM_RANGES 2
+
+/* Run test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int overlap_iosysid; /* The ID for iosystem of overlap_comm. */
+    MPI_Group world_group; /* An MPI group of world. */
+    MPI_Group overlap_group; /* An MPI group of 0, 1, and 3. */
+    MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */
+    int overlap_rank = -1; /* Tasks rank in communicator. */
+    int overlap_size = 0; /* Size of communicator. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+    
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+
+        /* Turn on logging. */
+        if ((ret = PIOc_set_log_level(3)))
+            return ret;
+
+        /* Initialize PIO system on world. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, 4, 1, 0, 1, &iosysid_world)))
+            ERR(ret);
+
+        /* Get MPI_Group of world comm. */
+        if ((ret = MPI_Comm_group(test_comm, &world_group)))
+            ERR(ret);
+
+        /* Create a group with tasks 0, 1, 3. */
+        int overlap_ranges[OVERLAP_NUM_RANGES][3] = {{0, 0, 1}, {1, 3, 2}};
+        if ((ret = MPI_Group_range_incl(world_group, OVERLAP_NUM_RANGES,
+                                        overlap_ranges, &overlap_group)))
+            ERR(ret);
+
+        /* Create a communicator from the overlap_group. */
+        if ((ret = MPI_Comm_create(test_comm, overlap_group, &overlap_comm)))
+            ERR(ret);
+
+        /* Learn my rank and the total number of processors in overlap
+         * group. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = MPI_Comm_rank(overlap_comm, &overlap_rank)))
+                MPIERR(ret);
+            if ((ret = MPI_Comm_size(overlap_comm, &overlap_size)))
+                MPIERR(ret);
+        }
+        printf("%d overlap_comm = %d overlap_rank = %d overlap_size = %d\n", my_rank,
+               overlap_comm, overlap_rank, overlap_size);
+
+        /* Initialize PIO system for overlap comm. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = PIOc_Init_Intracomm(overlap_comm, 1, 1, 0, 1, &overlap_iosysid)))
+                ERR(ret);
+        }
+
+        printf("%d pio finalizing %d\n", my_rank, overlap_iosysid);
+        /* Finalize PIO system. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            printf("%d calling PIOc_finalize with iosysid = %d\n", my_rank, overlap_iosysid);
+            if ((ret = PIOc_finalize(overlap_iosysid)))
+                ERR(ret);
+        }
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+        printf("%d pio finalized\n", my_rank);
+
+        /* Free MPI resources used by test. */
+        if ((ret = MPI_Group_free(&overlap_group)))
+            ERR(ret);
+        if ((ret = MPI_Group_free(&world_group)))
+            ERR(ret);
+        if (overlap_comm != MPI_COMM_NULL)
+            if ((ret = MPI_Comm_free(&overlap_comm)))
+                ERR(ret);
+        printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    }
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem3_simple2.c b/src/externals/pio2/tests/cunit/test_iosystem3_simple2.c
new file mode 100644
index 00000000000..973ef16c7eb
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem3_simple2.c
@@ -0,0 +1,101 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran
+ * pio_iosystem_tests3.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem3"
+
+/* Used when initializing PIO. */
+#define STRIDE1 1
+#define BASE0 0
+#define NUM_IO4 4
+#define REARRANGER 1
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    char fname0[NC_MAX_NAME + 1];
+    int ncid;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Initialize PIO system on world. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, REARRANGER, &iosysid_world)))
+            ERR(ret);
+
+        for (int i = 0; i < num_flavors; i++)
+        {
+            /* Create the file. */
+            sprintf(fname0, "test_iosystem3_simple2_%d.nc", i);
+            if ((ret = PIOc_createfile(iosysid_world, &ncid, &flavor[i], fname0, NC_CLOBBER)))
+                return ret;
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                return ret;
+
+            /* Close the file. */
+            if ((ret = PIOc_closefile(ncid)))
+                return ret;
+
+            /* Now check the first file from WORLD communicator. */
+            int mode = PIO_WRITE;
+
+            /* Open the file. */
+            if ((ret = PIOc_openfile(iosysid_world, &ncid, &flavor[i], fname0, mode)))
+                return ret;
+
+            /* Check the file. */
+            int ndims;
+            if ((ret = PIOc_inq(ncid, &ndims, NULL, NULL, NULL)))
+                return ret;
+
+            /* Close the file. */
+            if ((ret = PIOc_closefile(ncid)))
+                return ret;
+        } /* next iotype */
+
+        /* Finalize PIO systems. */
+        printf("%d pio finalizing\n", my_rank);
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc.c b/src/externals/pio2/tests/cunit/test_pioc.c
new file mode 100644
index 00000000000..3cffd645463
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc.c
@@ -0,0 +1,1885 @@
+/*
+ * Tests for PIO Functions.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 40
+#define Y_DIM_LEN 40
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 1
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* The name of the attribute in the netCDF output files. */
+#define ATT_NAME "foo"
+#define ATT_NAME2 "bar"
+
+/* Value to write to attributes. */
+#define ATT_VAL 42
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Values for some netcdf-4 settings. */
+#define VAR_CACHE_SIZE (1024 * 1024)
+#define VAR_CACHE_NELEMS 10
+#define VAR_CACHE_PREEMPTION 0.5
+
+/* Number of NetCDF classic types. */
+#define NUM_CLASSIC_TYPES 6
+
+/* Number of NetCDF-4 types. */
+#define NUM_NETCDF4_TYPES 12
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* Length of chunksizes to use in netCDF-4 files. */
+PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+#define DIM_LEN 4
+
+/* Length of the max maplen in decomp testing. */
+#define MAX_MAPLEN 1
+
+
+/* Create the decomposition to divide the 1-dimensional sample data
+ * between the 4 tasks.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim1_len the length of the dimension.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len, int *ioid)
+{
+#define NDIM1 1
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int dim_len[NDIM1] = {dim1_len};
+    int bad_dim_len[NDIM1] = {-50};
+    int ret;
+
+    /* How many data elements per task? */
+    elements_per_pe = dim1_len / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* These should fail. */
+    if (PIOc_InitDecomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe,
+                        compdof, ioid, NULL, NULL, NULL) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe,
+                        compdof, ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/* Check the contents of the test file. */
+int check_darray_file(int iosysid, int ntasks, int my_rank, char *filename)
+{
+    int ncid;
+    int ndims, nvars, ngatts, unlimdimid;
+    char dim_name_in[PIO_MAX_NAME + 1];
+    PIO_Offset dim_len_in;
+    PIO_Offset arraylen = 1;
+    float data_in;
+    int ioid;
+    int ret;
+
+    assert(filename);
+
+    /* Open the file. */
+    if ((ret = PIOc_open(iosysid, filename, NC_NOWRITE, &ncid)))
+        return ret;
+
+    /* Check metadata. */
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != 1 || nvars != 1 || ngatts != 0 || unlimdimid != -1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)))
+        return ret;
+    if (strcmp(dim_name_in, DIM_NAME) || dim_len_in != DIM_LEN)
+        return ERR_WRONG;
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(ntasks, my_rank, iosysid, DIM_LEN, &ioid)))
+        return ret;
+
+    /* Read data. */
+    if ((ret = PIOc_read_darray(ncid, 0, ioid, arraylen, &data_in)))
+        return ret;
+
+    /* Check data. */
+    if (data_in != my_rank * 10)
+        return ERR_WRONG;
+
+    /* Close the file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Test the darray functionality. */
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int fv)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dim_len[NDIM1] = {DIM_LEN}; /* Length of the dimensions in the sample data. */
+    int dimids[NDIM1];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "%s_flavor_%d_fv_%d.nc", TEST_NAME, flavor[fmt], fv);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+               flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        if ((ret = PIOc_def_dim(ncid, DIM_NAME, (PIO_Offset)dim_len[0], &dimids[0])))
+            ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM1, dimids, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Write some data. */
+        PIO_Offset arraylen = 1;
+        float fillvalue = 0.0;
+        float *fillvaluep = fv ? &fillvalue : NULL;
+        float test_data[arraylen];
+        for (int f = 0; f < arraylen; f++)
+            test_data[f] = my_rank * 10 + f;
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvaluep)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("rank: %d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check the file contents. */
+        if ((ret = check_darray_file(iosysid, TARGET_NTASKS, my_rank, filename)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/* Check the dimension names.
+ *
+ * @param my_rank rank of process
+ * @param ncid ncid of open netCDF file
+ * @returns 0 for success, error code otherwise. */
+int check_dim_names(int my_rank, int ncid, MPI_Comm test_comm)
+{
+    char dim_name[PIO_MAX_NAME + 1];
+    char zero_dim_name[PIO_MAX_NAME + 1];
+    int my_test_rank;
+    int ret;
+
+    /* Find rank in test communicator. */
+    if ((ret = MPI_Comm_rank(test_comm, &my_test_rank)))
+        MPIERR(ret);
+
+    for (int d = 0; d < NDIM; d++)
+    {
+        memset(dim_name, 0, sizeof(dim_name));
+        if ((ret = PIOc_inq_dimname(ncid, d, dim_name)))
+            return ret;
+        printf("my_rank %d my_test_rank %d dim %d name %s\n", my_rank, my_test_rank, d, dim_name);
+
+        /* Did other ranks get the same name? */
+        memset(zero_dim_name, 0, sizeof(zero_dim_name));
+        if (!my_test_rank)
+            strcpy(zero_dim_name, dim_name);
+        printf("rank %d dim_name %s zero_dim_name %s\n", my_rank, dim_name, zero_dim_name);
+        if ((ret = MPI_Bcast(&zero_dim_name, strlen(dim_name) + 1, MPI_CHAR, 0,
+                             test_comm)))
+            MPIERR(ret);
+        printf("%d zero_dim_name = %s dim_name = %s\n", my_rank, zero_dim_name, dim_name);
+        if (strcmp(dim_name, zero_dim_name))
+            return ERR_AWFUL;
+    }
+    return 0;
+}
+
+/* Check the variable name.
+ *
+ * @param my_rank rank of process
+ * @param ncid ncid of open netCDF file
+ *
+ * @returns 0 for success, error code otherwise. */
+int check_var_name(int my_rank, int ncid, MPI_Comm test_comm)
+{
+    char var_name[PIO_MAX_NAME + 1];
+    char zero_var_name[PIO_MAX_NAME + 1];
+    int my_test_rank;
+    int ret;
+
+    /* Find rank in test communicator. */
+    if ((ret = MPI_Comm_rank(test_comm, &my_test_rank)))
+        MPIERR(ret);
+
+    memset(var_name, 0, sizeof(var_name));
+    if ((ret = PIOc_inq_varname(ncid, 0, var_name)))
+        return ret;
+    printf("my_rank %d var name %s\n", my_rank, var_name);
+
+    /* Did other ranks get the same name? */
+    memset(zero_var_name, 0, sizeof(zero_var_name));
+    if (!my_test_rank)
+        strcpy(zero_var_name, var_name);
+    if ((ret = MPI_Bcast(&zero_var_name, strlen(var_name) + 1, MPI_CHAR, 0,
+                         test_comm)))
+        MPIERR(ret);
+    if (strcmp(var_name, zero_var_name))
+        return ERR_AWFUL;
+    return 0;
+}
+
+/* Check the attribute name.
+ *
+ * @param my_rank rank of process
+ * @param ncid ncid of open netCDF file
+ * @param flavor the iotype
+ * @param test_comm the MPI communicator of the test.
+ * @param async 1 if we are testing async, 0 otherwise.
+ * @returns 0 for success, error code otherwise. 
+ */
+int check_atts(int my_rank, int ncid, int flavor, MPI_Comm test_comm, int async)
+{
+    char att_name[PIO_MAX_NAME + 1];
+    char zero_att_name[PIO_MAX_NAME + 1];
+    int my_test_rank;
+    nc_type att_type;
+    PIO_Offset att_len;
+    signed char att_schar_value;
+    short att_short_value;
+    int att_int_value;
+    float att_float_value;
+    double att_double_value;
+    unsigned char att_uchar_value;
+    unsigned short att_ushort_value;
+    unsigned int att_uint_value;
+    long long int att_int64_value;
+    unsigned long long int att_uint64_value;
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;
+
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    /* Find rank in test communicator. */
+    if ((ret = MPI_Comm_rank(test_comm, &my_test_rank)))
+        MPIERR(ret);
+
+    memset(att_name, 0, sizeof(att_name));
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, att_name)))
+        return ret;
+
+    /* Did all ranks get the same name? */
+    memset(zero_att_name, 0, sizeof(zero_att_name));
+    if (!my_test_rank)
+        strcpy(zero_att_name, att_name);
+    if ((ret = MPI_Bcast(&zero_att_name, strlen(att_name) + 1, MPI_CHAR, 0,
+                         test_comm)))
+        MPIERR(ret);
+    if (strcmp(att_name, zero_att_name))
+        return ERR_AWFUL;
+
+    /* These should not work. */
+    if (PIOc_get_att_int(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &att_int_value) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_get_att_int(ncid, TEST_VAL_42, ATT_NAME, &att_int_value) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_get_att_int(ncid, NC_GLOBAL, NULL, &att_int_value) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* These should not work. */
+    if (PIOc_inq_att(ncid, NC_GLOBAL, too_long_name, &att_type, &att_len) != PIO_EINVAL)
+        return ERR_WRONG;
+    int tmp_attid;
+    if (PIOc_inq_attid(ncid, NC_GLOBAL, too_long_name, &tmp_attid) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Check first att. */
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &att_type, &att_len)))
+        return ret;
+    if (att_type != PIO_INT || att_len != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_int_value)))
+        return ret;
+    if (att_int_value != ATT_VAL)
+        return ERR_WRONG;
+
+    /* Try the typeless get. */
+    int att_int_value2;
+    if ((ret = PIOc_get_att(ncid, NC_GLOBAL, ATT_NAME, &att_int_value2)))
+        ERR(ret);
+    if (att_int_value2 != ATT_VAL)
+        return ERR_WRONG;
+    
+    /* Check second att. */
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME2, &att_type, &att_len)))
+        return ret;
+    if (att_type != PIO_FLOAT || att_len != 1)
+        return ERR_WRONG;
+
+    /* Try converting to every type. */
+    if ((ret = PIOc_get_att_schar(ncid, NC_GLOBAL, ATT_NAME2, &att_schar_value)))
+        return ret;
+    if (att_schar_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, ATT_NAME2, &att_short_value)))
+        return ret;
+    if (att_short_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME2, &att_int_value)))
+        return ret;
+    if (att_int_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, ATT_NAME2, &att_float_value)))
+        return ret;
+    if (att_float_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, ATT_NAME2, &att_double_value)))
+        return ret;
+    if (att_double_value != ATT_VAL)
+        return ERR_WRONG;
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_att_uchar(ncid, NC_GLOBAL, ATT_NAME2, &att_uchar_value)))
+            return ret;
+        if (att_uchar_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_ushort(ncid, NC_GLOBAL, ATT_NAME2, &att_ushort_value)))
+            return ret;
+        if (att_ushort_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_uint(ncid, NC_GLOBAL, ATT_NAME2, &att_uint_value)))
+            return ret;
+        if (att_uint_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_longlong(ncid, NC_GLOBAL, ATT_NAME2, &att_int64_value)))
+            return ret;
+        if (att_int64_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_ulonglong(ncid, NC_GLOBAL, ATT_NAME2, &att_uint64_value)))
+            return ret;
+        if (att_uint64_value != ATT_VAL)
+            return ERR_WRONG;
+    }
+    return 0;
+}
+
+/*
+ * Check error strings.
+ *
+ * @param my_rank rank of this task.
+ * @param num_tries number of errcodes to try.
+ * @param errcode pointer to array of error codes, of length num_tries.
+ * @param expected pointer to an array of strings, with the expected
+ * error messages for each error code.
+ * @returns 0 for success, error code otherwise.
+ */
+int check_error_strings(int my_rank, int num_tries, int *errcode,
+                        const char **expected)
+{
+    int ret;
+
+    /* Try each test code. */
+    for (int try = 0; try < num_tries; try++)
+    {
+        char errstr[PIO_MAX_NAME + 1];
+
+        /* Get the error string for this errcode. */
+        if ((ret = PIOc_strerror(errcode[try], errstr)))
+            return ret;
+
+        printf("%d for errcode = %d message = %s\n", my_rank, errcode[try], errstr);
+
+        /* Check that it was as expected. */
+        if (strncmp(errstr, expected[try], strlen(expected[try])))
+        {
+            printf("%d expected %s got %s\n", my_rank, expected[try], errstr);
+            return ERR_AWFUL;
+        }
+        if (!my_rank)
+            printf("%d errcode = %d passed\n", my_rank, errcode[try]);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_iotype_available() function.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int test_iotypes(int my_rank)
+{
+    /* This is never present. */
+    if (PIOc_iotype_available(1000))
+        return ERR_WRONG;
+    
+    /* NetCDF is always present. */
+    if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF))
+        return ERR_WRONG;
+
+    /* Pnetcdf may or may not be present. */
+#ifdef _PNETCDF
+    if (!PIOc_iotype_available(PIO_IOTYPE_PNETCDF))
+        return ERR_WRONG;
+#else
+    if (PIOc_iotype_available(PIO_IOTYPE_PNETCDF))
+        return ERR_WRONG;
+#endif /* _PNETCDF */
+
+    /* NetCDF-4 may or may not be present. */
+#ifdef _NETCDF4
+    if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF4C))
+        return ERR_WRONG;
+    if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF4P))
+        return ERR_WRONG;
+#else
+    if (PIOc_iotype_available(PIO_IOTYPE_NETCDF4C))
+        return ERR_WRONG;
+    if (PIOc_iotype_available(PIO_IOTYPE_NETCDF4P))
+        return ERR_WRONG;
+#endif /* _NETCDF4 */
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for classic netCDF.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_netcdf(int my_rank)
+{
+#define NUM_NETCDF_TRIES 5
+    int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, NC4_LAST_ERROR - 1, 0, 1, -600};
+    const char *expected[NUM_NETCDF_TRIES] = {"NetCDF: Not a valid ID",
+                                              "Unknown Error: Unrecognized error code", "No error",
+                                              nc_strerror(1), "Unknown Error: Unrecognized error code"};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_NETCDF_TRIES, errcode, expected)))
+        return ret;
+
+    /* When called with a code of 0, these functions should do nothing
+     * and return 0. */
+    if (check_mpi(NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (check_mpi2(NULL, NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (pio_err(NULL, NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (check_netcdf(NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (check_netcdf2(NULL, NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+
+    /* When called with other error messages, these functions should
+     * return PIO_EIO. */
+    if (check_mpi(NULL, MPI_ERR_OTHER, __FILE__, __LINE__) != PIO_EIO)
+        return ERR_WRONG;
+    if (check_mpi(NULL, MPI_ERR_UNKNOWN, __FILE__, __LINE__) != PIO_EIO)
+        return ERR_WRONG;
+
+    if (!my_rank)
+        printf("check_strerror_netcdf SUCCEEDED!\n");
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for netCDF-4.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_netcdf4(int my_rank)
+{
+#ifdef _NETCDF4
+#define NUM_NETCDF4_TRIES 2
+    int errcode[NUM_NETCDF4_TRIES] = {NC_ENOTNC3, NC_ENOPAR};
+    const char *expected[NUM_NETCDF4_TRIES] =
+        {"NetCDF: Attempting netcdf-3 operation on netcdf-4 file",
+         "NetCDF: Parallel operation on file opened for non-parallel access"};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_NETCDF4_TRIES, errcode, expected)))
+        return ret;
+
+    if (!my_rank)
+        printf("check_strerror_netcdf4 SUCCEEDED!\n");
+#endif /* _NETCDF4 */
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for parallel-netCDF.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_pnetcdf(int my_rank)
+{
+#ifdef _PNETCDF
+#define NUM_PNETCDF_TRIES 2
+    int errcode[NUM_PNETCDF_TRIES] = {NC_EMULTIDEFINE_VAR_NUM, NC_EMULTIDEFINE_ATTR_VAL};
+    const char *expected[NUM_PNETCDF_TRIES] =
+        {"Number of variables is",
+         "Attribute value is inconsistent among processes."};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_PNETCDF_TRIES, errcode, expected)))
+        return ret;
+
+    if (!my_rank)
+        printf("check_strerror_pnetcdf SUCCEEDED!\n");
+#endif /* _PNETCDF */
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for PIO.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_pio(int my_rank)
+{
+#define NUM_PIO_TRIES 6
+    int errcode[NUM_PIO_TRIES] = {PIO_EBADID,
+                                  NC_ENOTNC3, NC4_LAST_ERROR - 1, 0, 1,
+                                  PIO_EBADIOTYPE};
+    const char *expected[NUM_PIO_TRIES] = {"NetCDF: Not a valid ID",
+                                           "NetCDF: Attempting netcdf-3 operation on netcdf-4 file",
+                                           "Unknown Error: Unrecognized error code", "No error",
+                                           nc_strerror(1), "Bad IO type"};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_PIO_TRIES, errcode, expected)))
+        return ret;
+
+    if (!my_rank)
+        printf("check_strerror_pio SUCCEEDED!\n");
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror(int my_rank)
+{
+    int ret;
+
+    printf("checking strerror for netCDF-classic error codes...\n");
+    if ((ret = check_strerror_netcdf(my_rank)))
+        return ret;
+
+    printf("checking strerror for netCDF-4 error codes...\n");
+    if ((ret = check_strerror_netcdf4(my_rank)))
+        return ret;
+
+    printf("checking strerror for pnetcdf error codes...\n");
+    if ((ret = check_strerror_pnetcdf(my_rank)))
+        return ret;
+
+    printf("checking strerror for PIO error codes...\n");
+    if ((ret = check_strerror_pio(my_rank)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Define metadata for the test file. */
+int define_metadata(int ncid, int my_rank, int flavor)
+{
+    int dimids[NDIM]; /* The dimension IDs. */
+    int varid; /* The variable ID. */
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;
+
+    /* Check invalid parameters. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+    if (PIOc_def_dim(ncid + 1, dim_name[0], (PIO_Offset)dim_len[0], &dimids[0]) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_def_dim(ncid, NULL, (PIO_Offset)dim_len[0], &dimids[0]) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_dim(ncid, too_long_name, (PIO_Offset)dim_len[0], &dimids[0]) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Define dimensions. */
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            return ret;
+
+    /* Check invalid parameters. */
+    if (PIOc_def_var(ncid + 1, VAR_NAME, PIO_INT, NDIM, dimids, &varid) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, NULL, PIO_INT, NDIM, dimids, &varid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, too_long_name, PIO_INT, NDIM, dimids, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, too_long_name, TEST_VAL_42, NDIM, dimids, &varid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, too_long_name, PIO_LONG_INTERNAL, NDIM, dimids, &varid) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Define a variable. */
+    if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid)))
+        return ret;
+
+    /* Set the fill mode. */
+    int fillmode = PIO_NOFILL;
+    int temp_mode;
+    int old_mode;
+    if ((ret = PIOc_set_fill(ncid, fillmode, &old_mode)))
+        return ERR_WRONG;
+    if ((ret = PIOc_set_fill(ncid, fillmode, &temp_mode)))
+        return ERR_WRONG;
+    printf("%d old_mode = %d temp_mode = %d\n", my_rank, old_mode, temp_mode);
+    if (temp_mode != PIO_NOFILL)
+        return ERR_WRONG;
+    if ((ret = PIOc_set_fill(ncid, old_mode, NULL)))
+        return ret;
+
+    /* Set the fill value for netCDF-4 files. */
+    int int_fill = -999;
+    int int_fill_in;
+    int fill_mode;
+
+    /* These should not work. */
+    if (PIOc_def_var_fill(ncid + TEST_VAL_42, varid, NC_FILL, &int_fill) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_def_var_fill(ncid, varid + TEST_VAL_42, NC_FILL, &int_fill) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_def_var_fill(ncid, varid, NC_FILL + TEST_VAL_42, &int_fill) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var_fill(ncid, varid, NC_FILL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    
+    /* Set the fill value. */
+    if ((ret = PIOc_def_var_fill(ncid, varid, NC_FILL, &int_fill)))
+        return ret;
+    
+    /* These should not work. */
+    if (PIOc_inq_var_fill(ncid + TEST_VAL_42, varid, &fill_mode, &int_fill_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_inq_var_fill(ncid, varid + TEST_VAL_42, &fill_mode, &int_fill_in) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    
+    /* Check the fill value. */
+    if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode, &int_fill_in)))
+        return ret;
+    if (fill_mode != NC_FILL || int_fill_in != int_fill)
+        ERR(ERR_WRONG);
+    
+    /* These should also work. */
+    int_fill_in = 0;
+    
+    /* This does not work for pnetcdf, but probably should. */
+    if (flavor != PIO_IOTYPE_PNETCDF)
+    {
+        if ((ret = PIOc_inq_var_fill(ncid, varid, NULL, &int_fill_in)))
+            return ret;
+        if (int_fill_in != int_fill)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_inq_var_fill(ncid, varid, NULL, NULL)))
+            return ret;
+    }
+    if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode, NULL)))
+        return ret;
+    if (fill_mode != NC_FILL)
+        ERR(ERR_WRONG);
+
+    return PIO_NOERR;
+}
+
+/* Check the metadata in the test file. */
+int check_metadata(int ncid, int my_rank, int flavor)
+{
+    int ndims, nvars, ngatts, unlimdimid, natts, dimid[NDIM];
+    PIO_Offset len_in;
+    char name_in[PIO_MAX_NAME + 1];
+    nc_type xtype_in;
+    int ret;
+
+    /* Check how many dims, vars, global atts there are, and the id of
+     * the unlimited dimension. */
+    if (PIOc_inq(ncid + 1, &ndims, &nvars, &ngatts, &unlimdimid) != PIO_EBADID)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        return ret;
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != NDIM || nvars != 1 || ngatts != 0 || unlimdimid != 0)
+        return ERR_AWFUL;
+
+    /* Check the dimensions. */
+    for (int d = 0; d < NDIM; d++)
+    {
+        if (PIOc_inq_dim(ncid + 1, d, name_in, &len_in) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_inq_dim(ncid, d + 40, name_in, &len_in) != PIO_EBADDIM)
+            return ERR_WRONG;
+        if ((ret = PIOc_inq_dim(ncid, d, NULL, NULL)))
+            return ret;
+        if ((ret = PIOc_inq_dim(ncid, d, name_in, &len_in)))
+            return ret;
+        if (len_in != dim_len[d] || strcmp(name_in, dim_name[d]))
+            return ERR_AWFUL;
+    }
+
+    /* Check the variable. */
+    if (PIOc_inq_var(ncid + 1, 0, name_in, &xtype_in, &ndims, dimid, &natts) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_inq_var(ncid, 45, name_in, &xtype_in, &ndims, dimid, &natts) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_var(ncid, 0, name_in, NULL, NULL, NULL, NULL)))
+        return ret;
+    if ((ret = PIOc_inq_var(ncid, 0, name_in, &xtype_in, &ndims, dimid, &natts)))
+        return ret;
+    if (strcmp(name_in, VAR_NAME) || xtype_in != PIO_INT || ndims != NDIM ||
+        dimid[0] != 0 || dimid[1] != 1 || dimid[2] != 2 || natts != 1)
+        return ERR_AWFUL;
+
+    return PIO_NOERR;
+}
+
+/* Test file operations.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @param test_comm the MPI communicator of the test.
+ * @param async 1 if we are testing async, 0 otherwise.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_names(int iosysid, int num_flavors, int *flavor, int my_rank,
+               MPI_Comm test_comm, int async)
+{
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;    /* Return code. */
+
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        int ncid;
+        int varid;
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+        int dimids[NDIM];        /* The dimension IDs. */
+        int att_val = ATT_VAL;
+        float float_att_val = ATT_VAL;
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s_names.nc", TEST_NAME, iotype_name);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            return ret;
+
+        /* Define netCDF dimensions and variable. */
+        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+        {
+            printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank,
+                   dim_name[d], dim_len[d]);
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                return ret;
+        }
+
+        /* These should not work. */
+        if (PIOc_setframe(ncid + TEST_VAL_42, 0, 0) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_setframe(ncid, -1, 0) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_advanceframe(ncid + TEST_VAL_42, 0) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_advanceframe(ncid, -1) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Check the dimension names. */
+        if ((ret = check_dim_names(my_rank, ncid, test_comm)))
+            ERR(ret);
+
+        /* These should not work. */
+        if (PIOc_put_att_int(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, PIO_INT, 1, &att_val) != PIO_EBADID)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, TEST_VAL_42, ATT_NAME, PIO_INT, 1, &att_val) != PIO_ENOTVAR)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, NULL, PIO_INT, 1, &att_val) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_INT, 1, NULL) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, too_long_name, PIO_INT, 1, &att_val) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_INT, -1, &att_val) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, TEST_VAL_42, 1, &att_val) != PIO_EBADTYPE)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_LONG_INTERNAL, 1, &att_val) != PIO_EBADTYPE)
+            ERR(ERR_WRONG);
+
+        /* Define a global attribute. */
+        if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_INT, 1, &att_val)))
+            ERR(ret);
+        if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, ATT_NAME2, PIO_FLOAT, 1, &float_att_val)))
+            ERR(ret);
+
+        /* Check the attribute name. */
+        if ((ret = check_atts(my_rank, ncid, flavor[fmt], test_comm, async)))
+            ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* Check the variable name. */
+        if ((ret = check_var_name(my_rank, ncid, test_comm)))
+            ERR(ret);
+
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("rank: %d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test file operations.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_files(int iosysid, int num_flavors, int *flavor, int my_rank)
+{
+    int ncid;
+    int ret;    /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+
+        /* Overwrite existing test file. */
+        int mode = PIO_CLOBBER;
+
+        /* If this is netCDF-4, add the netCDF4 flag. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            printf("%d adding NC_NETCDF4 flag\n", my_rank);
+            mode |= NC_NETCDF4;
+        }
+
+        /* If this is pnetcdf or netCDF-4 parallel, add the MPIIO flag. */
+        if (flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            printf("%d adding NC_MPIIO flag\n", my_rank);
+            mode |= NC_MPIIO;
+        }
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s.nc", TEST_NAME, iotype_name);
+
+        /* Testing some invalid parameters. */
+        if (PIOc_create(iosysid + 1, filename, mode, &ncid) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_create(iosysid, filename, mode, NULL) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_create(iosysid, NULL, mode, &ncid) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Create the netCDF output file. */
+        printf("%d Creating sample file %s with format %d...\n", my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_create(iosysid, filename, mode, &ncid)))
+            ERR(ret);
+
+        /* Check this support function. */
+        if (!PIOc_File_is_Open(ncid))
+            ERR(ERR_WRONG);
+        if (PIOc_File_is_Open(ncid + 1))
+            ERR(ERR_WRONG);
+
+        /* Define the test file metadata. */
+        if ((ret = define_metadata(ncid, my_rank, flavor[fmt])))
+            ERR(ret);
+
+        /* End define mode. */
+        if (PIOc_enddef(ncid + 1) != PIO_EBADID)
+            return ERR_WRONG;
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if (PIOc_closefile(ncid + 1) != PIO_EBADID)
+            return ERR_WRONG;
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check some invalid paramters. */
+        if (PIOc_open(iosysid + 1, filename, mode, &ncid) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_open(iosysid, NULL, mode, &ncid) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_open(iosysid, filename, mode, NULL) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Reopen the test file. */
+        printf("%d Re-opening sample file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_open(iosysid, filename, mode, &ncid)))
+            ERR(ret);
+
+        /* Check the test file metadata. */
+        if ((ret = check_metadata(ncid, my_rank, flavor[fmt])))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test the deletion of files.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_deletefile(int iosysid, int num_flavors, int *flavor, int my_rank)
+{
+    int ncid;
+    int ret;    /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+        int old_method;
+
+        /* These should fail. */
+        if (PIOc_set_iosystem_error_handling(iosysid + TEST_VAL_42, PIO_RETURN_ERROR, &old_method) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_set_iosystem_error_handling(iosysid, PIO_RETURN_ERROR + TEST_VAL_42, &old_method) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Set error handling. */
+        if ((ret = PIOc_set_iosystem_error_handling(iosysid, PIO_RETURN_ERROR, &old_method)))
+            return ret;
+        if (old_method != PIO_INTERNAL_ERROR && old_method != PIO_RETURN_ERROR)
+            return ERR_WRONG;
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "delete_me_%s_%s.nc", TEST_NAME, iotype_name);
+
+        printf("%d testing delete for file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        int bad_iotype = TEST_VAL_42;
+        if (PIOc_createfile(iosysid, &ncid, &bad_iotype, filename, PIO_CLOBBER) != PIO_EINVAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* This should not work. */
+        if (PIOc_deletefile(iosysid + TEST_VAL_42, filename) != PIO_EBADID)
+            ERR(ERR_WRONG);
+
+        /* Now delete the file. */
+        printf("%d Deleting %s...\n", my_rank, filename);
+        if ((ret = PIOc_deletefile(iosysid, filename)))
+            ERR(ret);
+
+        /* Make sure it is gone. Openfile will now return an error
+         * code when I try to open the file. */
+        if (!PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE))
+            ERR(ERR_WRONG);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test the netCDF-4 optimization functions. */
+int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank)
+{
+    int ncid;    /* The ncid of the netCDF file. */
+    int dimids[NDIM];    /* The dimension IDs. */
+    int varid;    /* The ID of the netCDF varable. */
+
+    /* For setting the chunk cache. */
+    PIO_Offset chunk_cache_size = 1024*1024;
+    PIO_Offset chunk_cache_nelems = 1024;
+    float chunk_cache_preemption = 0.5;
+
+    /* For reading the chunk cache. */
+    PIO_Offset chunk_cache_size_in;
+    PIO_Offset chunk_cache_nelems_in;
+    float chunk_cache_preemption_in;
+
+    int storage = NC_CHUNKED; /* Storage of netCDF-4 files (contiguous vs. chunked). */
+    PIO_Offset my_chunksize[NDIM]; /* Chunksizes we get from file. */
+    int shuffle;    /* The shuffle filter setting in the netCDF-4 test file. */
+    int deflate;    /* Non-zero if deflate set for the variable in the netCDF-4 test file. */
+    int deflate_level;    /* The deflate level set for the variable in the netCDF-4 test file. */
+    int endianness;    /* Endianness of variable. */
+    PIO_Offset var_cache_size;    /* Size of the var chunk cache. */
+    PIO_Offset var_cache_nelems; /* Number of elements in var cache. */
+    float var_cache_preemption;     /* Var cache preemption. */
+    char varname_in[PIO_MAX_NAME];
+    int expected_ret; /* The return code we expect to get. */
+    int ret;    /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s.nc", TEST_NAME, iotype_name);
+
+        printf("%d Setting chunk cache for file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+
+        /* Try to set the chunk cache. */
+        chunk_cache_preemption = 0.5;
+        ret = PIOc_set_chunk_cache(iosysid, flavor[fmt], chunk_cache_size,
+                                   chunk_cache_nelems, chunk_cache_preemption);
+
+        /* What result did we expect to get? */
+        expected_ret = flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P ?
+            PIO_NOERR : PIO_ENOTNC4;
+        if (ret != expected_ret)
+            ERR(ERR_AWFUL);
+
+        /* Try to set the chunk cache for netCDF-4 iotypes. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            if ((ret = PIOc_set_chunk_cache(iosysid, flavor[fmt], chunk_cache_size,
+                                            chunk_cache_nelems, chunk_cache_preemption)))
+                ERR(ERR_AWFUL);
+
+        /* Now check the chunk cache. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            if ((ret = PIOc_get_chunk_cache(iosysid, flavor[fmt], &chunk_cache_size_in,
+                                            &chunk_cache_nelems_in, &chunk_cache_preemption_in)))
+                ERR(ERR_AWFUL);
+
+            /* Check that we got the correct values. */
+            if (chunk_cache_size_in != chunk_cache_size || chunk_cache_nelems_in != chunk_cache_nelems ||
+                chunk_cache_preemption_in != chunk_cache_preemption)
+                ERR(ERR_AWFUL);
+        }
+
+        /* Create the netCDF output file. */
+        printf("%d Creating sample file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("%d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+        {
+            printf("%d Defining netCDF dimension %s, length %d\n", my_rank,
+                   dim_name[d], dim_len[d]);
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+        }
+        printf("%d Defining netCDF variable %s, ndims %d\n", my_rank, VAR_NAME, NDIM);
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* Check that invalid arguments are properly rejected. */
+        if (PIOc_def_var_chunking(ncid + TEST_VAL_42, 0, NC_CHUNKED, chunksize) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_inq_var_chunking(ncid + TEST_VAL_42, 0, &storage, my_chunksize) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_inq_var_deflate(ncid + TEST_VAL_42, 0, &shuffle, &deflate, &deflate_level) != PIO_EBADID)
+            ERR(ret);
+        if (PIOc_def_var_endian(ncid + TEST_VAL_42, 0, 1) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_def_var_deflate(ncid + TEST_VAL_42, 0, 0, 0, 0) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_inq_var_endian(ncid + TEST_VAL_42, 0, &endianness) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_set_var_chunk_cache(ncid + TEST_VAL_42, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                     VAR_CACHE_PREEMPTION) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_get_var_chunk_cache(ncid + TEST_VAL_42, 0, &var_cache_size, &var_cache_nelems,
+                                     &var_cache_preemption) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_set_chunk_cache(iosysid + TEST_VAL_42, flavor[fmt], chunk_cache_size, chunk_cache_nelems,
+                                 chunk_cache_preemption) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_get_chunk_cache(iosysid + TEST_VAL_42, flavor[fmt], &chunk_cache_size,
+                                 &chunk_cache_nelems, &chunk_cache_preemption) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            if (PIOc_def_var_chunking(ncid, TEST_VAL_42, NC_CHUNKED, chunksize) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_chunking(ncid, TEST_VAL_42, &storage, my_chunksize) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_var_chunk_cache(ncid, TEST_VAL_42, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                         VAR_CACHE_PREEMPTION) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_var_chunk_cache(ncid, TEST_VAL_42, &var_cache_size, &var_cache_nelems,
+                                         &var_cache_preemption) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+        }
+        else
+        {
+            printf("my ret = %d\n", PIOc_def_var_chunking(ncid, TEST_VAL_42, NC_CHUNKED, chunksize));
+            if (PIOc_def_var_chunking(ncid, TEST_VAL_42, NC_CHUNKED, chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_chunking(ncid, TEST_VAL_42, &storage, my_chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_var_chunk_cache(ncid, TEST_VAL_42, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                         VAR_CACHE_PREEMPTION) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_var_chunk_cache(ncid, TEST_VAL_42, &var_cache_size, &var_cache_nelems,
+                                         &var_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+        }
+
+        /* For netCDF-4 files, set the chunksize to improve performance. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            unsigned long long too_big_chunksize[NDIM] = {(unsigned long long)NC_MAX_INT64 + TEST_VAL_42, X_DIM_LEN/2, Y_DIM_LEN/2};
+            if (PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, (MPI_Offset *)too_big_chunksize) == PIO_NOERR)
+                ERR(ret);
+
+            printf("%d Defining chunksizes\n", my_rank);
+            if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)))
+                ERR(ret);
+
+            /* Setting deflate should not work with parallel iotype. */
+            printf("%d Defining deflate\n", my_rank);
+            ret = PIOc_def_var_deflate(ncid, 0, 0, 1, 1);
+            if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            {
+                if (ret == PIO_NOERR)
+                    ERR(ERR_WRONG);
+            }
+            else
+            {
+                if (ret != PIO_NOERR)
+                    ERR(ERR_WRONG);
+            }
+
+            /* Check that the inq_varname function works. */
+            printf("%d Checking varname\n", my_rank);
+            if ((ret = PIOc_inq_varname(ncid, 0, NULL)))
+                ERR(ret);
+            if ((ret = PIOc_inq_varname(ncid, 0, varname_in)))
+                ERR(ret);
+
+            /* Check that the inq_var_chunking function works. */
+            printf("%d Checking chunksizes\n", my_rank);
+            if ((ret = PIOc_inq_var_chunking(ncid, 0, NULL, NULL)))
+                ERR(ret);
+            if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)))
+                ERR(ret);
+
+            /* Check the answers. */
+            if (storage != NC_CHUNKED)
+                ERR(ERR_AWFUL);
+            for (int d1 = 0; d1 < NDIM; d1++)
+                if (my_chunksize[d1] != chunksize[d1])
+                    ERR(ERR_AWFUL);
+
+            /* Check that the inq_var_deflate functions works. */
+            if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)))
+                ERR(ret);
+
+            /* For serial netCDF-4 deflate is turned on by default */
+            if (flavor[fmt] == PIO_IOTYPE_NETCDF4C)
+                if (shuffle || !deflate || deflate_level != 1)
+                    ERR(ERR_AWFUL);
+
+            /* For parallel netCDF-4, no compression available. :-( */
+            if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+                if (shuffle || deflate)
+                    ERR(ERR_AWFUL);
+
+            /* Check setting the chunk cache for the variable. */
+            printf("%d PIOc_set_var_chunk_cache...\n", my_rank);
+            if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                                VAR_CACHE_PREEMPTION)))
+                ERR(ret);
+
+            /* Check getting the chunk cache values for the variable. */
+            printf("%d PIOc_get_var_chunk_cache...\n", my_rank);
+            if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
+                                                &var_cache_preemption)))
+                ERR(ret);
+
+            /* Check that we got expected values. */
+            printf("%d var_cache_size = %lld\n", my_rank, var_cache_size);
+            if (var_cache_size != VAR_CACHE_SIZE)
+                ERR(ERR_AWFUL);
+            if (var_cache_nelems != VAR_CACHE_NELEMS)
+                ERR(ERR_AWFUL);
+            if (var_cache_preemption != VAR_CACHE_PREEMPTION)
+                ERR(ERR_AWFUL);
+
+            if ((ret = PIOc_def_var_endian(ncid, 0, 1)))
+                ERR(ERR_AWFUL);
+            if ((ret = PIOc_inq_var_endian(ncid, 0, &endianness)))
+                ERR(ERR_AWFUL);
+            if (endianness != 1)
+                ERR(ERR_WRONG);
+        }
+        else
+        {
+            /* Trying to set or inq netCDF-4 settings for non-netCDF-4
+             * files results in the PIO_ENOTNC4 error. */
+            if (PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level) != PIO_ENOTNC4)
+                ERR(ret);
+            if (PIOc_def_var_endian(ncid, 0, 1) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_endian(ncid, 0, &endianness) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                         VAR_CACHE_PREEMPTION) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
+                                         &var_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_chunk_cache(iosysid, flavor[fmt], chunk_cache_size, chunk_cache_nelems,
+                                     chunk_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_chunk_cache(iosysid, flavor[fmt], &chunk_cache_size,
+                                     &chunk_cache_nelems, &chunk_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+        }
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/** Test the malloc_iodesc() function.
+ *
+ * @param my_rank rank of this task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_malloc_iodesc2(int iosysid, int my_rank)
+{
+    /* More types are available for netCDF-4 builds. */
+#ifdef _NETCDF4
+    int num_types = NUM_NETCDF_TYPES;
+#else
+    int num_types = NUM_CLASSIC_TYPES;
+#endif /* _NETCDF4 */
+    int test_type[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                       PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING};
+    int mpi_type[NUM_NETCDF_TYPES] = {MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, MPI_FLOAT, MPI_DOUBLE,
+                                      MPI_UNSIGNED_CHAR, MPI_UNSIGNED_SHORT, MPI_UNSIGNED, MPI_LONG_LONG,
+                                      MPI_UNSIGNED_LONG_LONG, MPI_CHAR};
+    int ioid;
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int ret;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Test with each type. */
+    for (int t = 0; t < num_types; t++)
+    {
+        if ((ret = malloc_iodesc(ios, test_type[t], 1, &iodesc)))
+            return ret;
+        if (iodesc->basetype != mpi_type[t])
+            return ERR_WRONG;
+        if (iodesc->ndims != 1)
+            return ERR_WRONG;
+        ioid = pio_add_to_iodesc_list(iodesc);
+        if (iodesc->firstregion)
+            free_region_list(iodesc->firstregion);
+        if ((ret = pio_delete_iodesc_from_list(ioid)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Test some decomp internal functions. */
+int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len,
+                         MPI_Comm test_comm, int async)
+{
+    int ioid;
+    char filename[NC_MAX_NAME + 1];    /* Test decomp filename. */
+    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(filename, "decomp_%s_rank_%d_async_%d.txt", TEST_NAME, my_rank, async);
+    sprintf(nc_filename, "nc_decomp_internal_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid)))
+        return ret;
+
+    /* Write the decomp file (on appropriate tasks). */
+    if ((ret = PIOc_write_decomp(filename, iosysid, ioid, test_comm)))
+        return ret;
+
+    /* Some values for the netCDF decomp file for this iosystem. */
+    char *title = "Very Simple Test Decompositon";
+    char *history = "Added to PIO automatic testing by Ed in February 2017.";
+    int global_dimlen[] = {DIM_LEN};
+    int task_maplen[TARGET_NTASKS] = {1, 1, 1, 1};
+    int map[TARGET_NTASKS][1] = {{0},{1},{2},{3}};
+
+    /* These should not work. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+    if (pioc_write_nc_decomp_int(iosysid + TEST_VAL_42, nc_filename, 0, NDIM1, global_dimlen,
+                                 TARGET_NTASKS, task_maplen, (int *)map, title,
+                                 history, 0) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pioc_write_nc_decomp_int(iosysid, NULL, 0, NDIM1, global_dimlen,
+                                 TARGET_NTASKS, task_maplen, (int *)map, title,
+                                 history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, NULL,
+                                 TARGET_NTASKS, task_maplen, (int *)map, title,
+                                 history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen,
+                                 TARGET_NTASKS, NULL, (int *)map, title,
+                                 history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen,
+                                 TARGET_NTASKS, task_maplen, (int *)map, too_long_name,
+                                 history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen,
+                                 TARGET_NTASKS, task_maplen, (int *)map, title,
+                                 too_long_name, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    
+
+    /* Write the decomposition file. */
+    if ((ret = pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen,
+                                        TARGET_NTASKS, task_maplen, (int *)map, title,
+                                        history, 0)))
+        return ret;
+
+    int ndims_in;
+    int num_tasks_in;
+    int max_maplen_in;
+    char title_in[PIO_MAX_NAME + 1];
+    char history_in[PIO_MAX_NAME + 1];
+    char source_in[PIO_MAX_NAME + 1];
+    char version_in[PIO_MAX_NAME + 1];
+    char expected_source[] = "Decomposition file produced by PIO library.";
+    int *global_dimlen_in;    
+    int *task_maplen_in;
+    int *map_in;
+    int fortran_order_in;
+
+    /* These should not work. */
+    if (pioc_read_nc_decomp_int(iosysid + TEST_VAL_42, nc_filename, &ndims_in, &global_dimlen_in,
+                                &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                history_in, source_in, version_in, &fortran_order_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pioc_read_nc_decomp_int(iosysid, NULL, &ndims_in, &global_dimlen_in,
+                                &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                history_in, source_in, version_in, &fortran_order_in) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (!pioc_read_nc_decomp_int(iosysid, "no_file", &ndims_in, &global_dimlen_in,
+                                &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                history_in, source_in, version_in, &fortran_order_in))
+        return ERR_WRONG;
+
+    /* Read the decomp file. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+
+    /* Did we get the correct answers? */
+    printf("source_in = %s\n", source_in);
+    if (strcmp(title, title_in) || strcmp(history, history_in) ||
+        strcmp(source_in, expected_source))
+        return ERR_WRONG;
+    if (ndims_in != NDIM1 || num_tasks_in != TARGET_NTASKS || max_maplen_in != 1 ||
+        fortran_order_in)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims_in; d++)
+        if (global_dimlen_in[d] != global_dimlen[d])
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        if (task_maplen_in[t] != 1)
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        for (int l = 0; l < max_maplen_in; l++)
+            if (map_in[t * max_maplen_in + l] != map[t][l])
+                return ERR_WRONG;
+
+    /* Free resources. */
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    /* These should also work. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, NULL, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, NULL,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       NULL, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, NULL, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, NULL, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, NULL, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, NULL,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       NULL, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, NULL, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, NULL, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, NULL)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Test some decomp public API functions. */
+int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len,
+                       MPI_Comm test_comm, int async)
+{
+    int ioid;
+    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    int ret;
+    
+    /* This will be our file name for writing out decompositions. */
+    sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid)))
+        return ret;
+
+    /* We will document our decomp file with metadata, like good
+     * netCDF users should. */
+    char *title = "Very Simple Test Decompositon";
+    char *history = "Added to PIO automatic testing by Ed in February 2017.";
+
+    /* These should not work. */
+    if (PIOc_write_nc_decomp(iosysid + TEST_VAL_42, nc_filename, 0, ioid, test_comm, title, history, 0) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_write_nc_decomp(iosysid, NULL, 0, ioid, test_comm, title, history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid + TEST_VAL_42, test_comm, title, history, 0) != PIO_EBADID)
+        return ERR_WRONG;
+
+    /* Write a netCDF decomp file for this iosystem. */
+    if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, test_comm, title, history, 0)))
+        return ret;
+
+    int ioid_in;
+    char title_in[PIO_MAX_NAME + 1];
+    char history_in[PIO_MAX_NAME + 1];
+    int fortran_order_in;
+
+    /* These should not work. */
+    if (PIOc_read_nc_decomp(iosysid + TEST_VAL_42, nc_filename, &ioid_in, test_comm, PIO_INT,
+                            title_in, history_in, &fortran_order_in) != PIO_EBADID)
+        return ret;
+    if (PIOc_read_nc_decomp(iosysid, NULL, &ioid_in, test_comm, PIO_INT, title_in,
+                            history_in, &fortran_order_in) != PIO_EINVAL)
+        return ret;
+    if (PIOc_read_nc_decomp(iosysid, nc_filename, NULL, test_comm, PIO_INT, title_in,
+                            history_in, &fortran_order_in) != PIO_EINVAL)
+        return ret;
+    
+    /* Read it using the public read function. */
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_INT,
+                                   title_in, history_in, &fortran_order_in)))
+        return ret;
+
+    /* Did we get expected results? */
+    if (strcmp(title, title_in) || strcmp(history, history_in))
+        return ERR_WRONG;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    /* These should also work. */
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_CHAR, NULL,
+                                   history_in, &fortran_order_in)))
+        return ret;
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_BYTE, title_in,
+                                   NULL, &fortran_order_in)))
+        return ret;
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_SHORT, title_in,
+                                  history_in, NULL)))
+        return ret;
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    /* Read it using the intertal function. */
+    int ndims_in;
+    int num_tasks_in;
+    int max_maplen_in;
+    char source_in[PIO_MAX_NAME + 1];
+    char version_in[PIO_MAX_NAME + 1];
+    char expected_source[] = "Decomposition file produced by PIO library.";
+    int *global_dimlen_in;
+    int *task_maplen_in;
+    int *map_in;
+
+    /* Read the decomp file. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+
+    /* Did we get the correct answers? */
+    printf("source_in = %s\n", source_in);
+    if (strcmp(title, title_in) || strcmp(history, history_in) ||
+        strcmp(source_in, expected_source))
+        return ERR_WRONG;
+    if (ndims_in != NDIM1 || num_tasks_in != TARGET_NTASKS || max_maplen_in != 1 ||
+        fortran_order_in)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims_in; d++)
+        if (global_dimlen_in[d] != DIM_LEN)
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        if (task_maplen_in[t] != 1)
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        for (int l = 0; l < max_maplen_in; l++)
+            if (map_in[t * max_maplen_in + l] != t)
+                return ERR_WRONG;
+
+    /* Free resources. */
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    /* /\* These should also work. *\/ */
+    /* if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, test_comm, title, history, 0))) */
+    /*     return ret; */
+    
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int ioid;
+    int my_test_size;
+    char filename[NC_MAX_NAME + 1];
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(filename, "decomp_%d.txt", my_rank);
+
+    /* Check iotypes. */
+    printf("%d Testing iotypes. async = %d\n", my_rank, async);
+    if ((ret = test_iotypes(my_rank)))
+        ERR(ret);
+
+    /* Test file deletes. */
+    printf("%d Testing deletefile. async = %d\n", my_rank, async);
+    if ((ret = test_deletefile(iosysid, num_flavors, flavor, my_rank)))
+        return ret;
+
+    /* Test file stuff. */
+    printf("%d Testing file creation. async = %d\n", my_rank, async);
+    if ((ret = test_files(iosysid, num_flavors, flavor, my_rank)))
+        return ret;
+
+    /* Test some misc stuff. */
+    if ((ret = test_malloc_iodesc2(iosysid, my_rank)))
+        return ret;
+
+    /* Test decomposition internal functions. */
+    if (!async)
+        if ((ret = test_decomp_internal(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async)))
+            return ret;
+
+    /* Test decomposition public API functions. */
+    if (!async)
+        if ((ret = test_decomp_public(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async)))
+            return ret;
+
+    if (!async)
+    {
+        printf("%d Testing darray. async = %d\n", my_rank, async);
+        
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition(my_test_size, my_rank, iosysid, DIM_LEN, &ioid)))
+            return ret;
+
+        /* Write out an ASCII version of the decomp file. */
+        printf("%d Calling write_decomp. async = %d\n", my_rank, async);
+        if ((ret = PIOc_write_decomp(filename, iosysid, ioid, test_comm)))
+            return ret;
+        printf("%d Called write_decomp. async = %d\n", my_rank, async);
+
+        /* Run the darray tests. */
+        for (int fv = 0; fv < 2; fv++)
+            if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, fv)))
+                return ret;
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    /* Check the error string function. */
+    printf("%d Testing streror. async = %d\n", my_rank, async);
+    if ((ret = check_strerror(my_rank)))
+        ERR(ret);
+
+    /* Test name stuff. */
+    printf("%d Testing names. async = %d\n", my_rank, async);
+    if ((ret = test_names(iosysid, num_flavors, flavor, my_rank, test_comm, async)))
+        return ret;
+
+    /* Test netCDF-4 functions. */
+    printf("%d Testing nc4 functions. async = %d\n", my_rank, async);
+    if ((ret = test_nc4(iosysid, num_flavors, flavor, my_rank)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run all tests. */
+int main(int argc, char **argv)
+{
+    /* Change the 5th arg to 3 to turn on logging. */
+    return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0,
+                         TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc_fill.c b/src/externals/pio2/tests/cunit/test_pioc_fill.c
new file mode 100644
index 00000000000..e2015dee7f0
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc_fill.c
@@ -0,0 +1,816 @@
+/*
+ * More tests for PIO data reading and writing routines.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc_fill"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The names of the variable in the netCDF output files. */
+#define VAR_NAME "Laurel"
+#define VAR_NAME_2 "Hardy"
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* 1D dim for fill_mode test. */
+#define DIM_NAME "SonsOfTheDesert"
+#define DIM_LEN 1
+
+/* Some sample data values to write. */
+char text[] = "hi";
+char char_data = 2;
+signed char byte_data = -42;
+short short_data = -300;
+int int_data = -10000;
+long int long_data = -20000;
+float float_data = -42.42;
+double double_data = -420000000000.5;
+unsigned char ubyte_data = 43;
+unsigned short ushort_data = 666;
+unsigned int uint_data = 666666;
+long long int64_data = -99999999999;
+unsigned long long uint64_data = 99999999999;
+
+char char_fill_value = 18;
+signed char byte_fill_value = -18;
+short short_fill_value = -1818;
+int int_fill_value = -18181818;
+float float_fill_value = 18.18;
+double double_fill_value = 1818181818.18;
+unsigned char ubyte_fill_value = 18;
+unsigned short ushort_fill_value = 1818;
+unsigned int uint_fill_value = 18181818;
+long long int64_fill_value = -181818181818;
+unsigned long long uint64_fill_value = 181818181818;
+
+char text_array[X_DIM_LEN/2][Y_DIM_LEN];
+char char_array[X_DIM_LEN/2][Y_DIM_LEN];
+signed char byte_array[X_DIM_LEN/2][Y_DIM_LEN];
+short short_array[X_DIM_LEN/2][Y_DIM_LEN];
+int int_array[X_DIM_LEN/2][Y_DIM_LEN];
+long int long_array[X_DIM_LEN/2][Y_DIM_LEN];
+float float_array[X_DIM_LEN/2][Y_DIM_LEN];
+double double_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned char ubyte_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned short ushort_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned int uint_array[X_DIM_LEN/2][Y_DIM_LEN];
+long long int64_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned long long uint64_array[X_DIM_LEN/2][Y_DIM_LEN];
+
+/* Fill up the data arrays with some values. */
+void init_arrays()
+{
+    for (int x = 0; x < X_DIM_LEN/2; x++)
+    {
+        strcpy(text_array[x], text);
+        for (int y = 0; y < Y_DIM_LEN; y++)
+        {
+            char_array[x][y] = char_data + y + x * Y_DIM_LEN;
+            byte_array[x][y] = byte_data + y + x * Y_DIM_LEN;
+            short_array[x][y] = short_data + y + x * Y_DIM_LEN;
+            int_array[x][y] = int_data + y + x * Y_DIM_LEN;
+            long_array[x][y] = int_data + y + x * Y_DIM_LEN;
+            float_array[x][y] = float_data + y + x * Y_DIM_LEN;
+            double_array[x][y] = double_data + y + x * Y_DIM_LEN;
+            ubyte_array[x][y] = ubyte_data + y + x * Y_DIM_LEN;
+            ushort_array[x][y] = ushort_data + y + x * Y_DIM_LEN;
+            uint_array[x][y] = uint_data + y + x * Y_DIM_LEN;
+            int64_array[x][y] = int64_data + y + x * Y_DIM_LEN;
+            uint64_array[x][y] = uint64_data + y + x * Y_DIM_LEN;
+        }
+    }
+}
+
+/* Use the vara functions to write some data to an open test file. */
+int putget_write_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                      int flavor)
+{
+    int ret;
+    printf("now writing data\n");
+
+    if ((ret = PIOc_put_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_text(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_int(ncid, varid[3], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_float(ncid, varid[4], start, count, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_double(ncid, varid[5], start, count, (double *)double_array)))
+        return ret;
+
+    printf("now wrote classic data\n");
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        printf("now writing netcdf4 data\n");
+        if ((ret = PIOc_put_vara_uchar(ncid, varid[6], start, count, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ushort(ncid, varid[7], start, count, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_uint(ncid, varid[8], start, count, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_longlong(ncid, varid[9], start, count, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ulonglong(ncid, varid[10], start, count, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+int check_fill(int ncid, int *varid, int flavor, int default_fill)
+{
+    int fill_mode;
+    char char_fill_value_in;
+    signed char byte_fill_value_in;
+    short short_fill_value_in;
+    int int_fill_value_in;
+    float float_fill_value_in;
+    double double_fill_value_in;
+    unsigned char ubyte_fill_value_in;
+    unsigned short ushort_fill_value_in;
+    unsigned int uint_fill_value_in;
+    long long int64_fill_value_in;
+    unsigned long long uint64_fill_value_in;
+    int ret;
+
+    printf("checking fill values for flavor %d default_fill %d\n", flavor, default_fill);
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[0], &fill_mode, &byte_fill_value_in)))
+        return ret;
+    printf("byte_fill_value_in = %d\n", (int)byte_fill_value_in);
+    if (fill_mode != NC_FILL || byte_fill_value_in != (default_fill ? NC_FILL_BYTE : byte_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[1], &fill_mode, &char_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || char_fill_value_in != (default_fill ? NC_FILL_CHAR : char_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[2], &fill_mode, &short_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || short_fill_value_in != (default_fill ? NC_FILL_SHORT : short_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[3], &fill_mode, &int_fill_value_in)))
+        return ret;
+    printf("int_fill_value_in = %d\n", int_fill_value_in);
+    if (fill_mode != NC_FILL || int_fill_value_in != (default_fill ? NC_FILL_INT : int_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[4], &fill_mode, &float_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || float_fill_value_in != (default_fill ? NC_FILL_FLOAT : float_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[5], &fill_mode, &double_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || double_fill_value_in != (default_fill ? NC_FILL_DOUBLE : double_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_inq_var_fill(ncid, varid[6], &fill_mode, &ubyte_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || ubyte_fill_value_in != (default_fill ? NC_FILL_UBYTE : ubyte_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[7], &fill_mode, &ushort_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || ushort_fill_value_in != (default_fill ? NC_FILL_USHORT : ushort_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[8], &fill_mode, &uint_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || uint_fill_value_in != (default_fill ? NC_FILL_UINT : uint_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[9], &fill_mode, &int64_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || int64_fill_value_in != (default_fill ? NC_FILL_INT64 : int64_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[10], &fill_mode, &uint64_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || uint64_fill_value_in != (default_fill ? NC_FILL_UINT64 : uint64_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+    }
+
+    return 0;
+}
+
+/* Use the vara functions to read some data from an open test file. */
+int putget_read_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                     int default_fill, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* Read the data we wrote. */
+    if ((ret = PIOc_get_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_text(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_short(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_int(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_float(ncid, varid[4], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_double(ncid, varid[5], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN/2; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara_uchar(ncid, varid[6], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ushort(ncid, varid[7], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_uint(ncid, varid[8], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_longlong(ncid, varid[9], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ulonglong(ncid, varid[10], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN/2; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    /* Check some fill value stuff. */
+    if ((ret = check_fill(ncid, varid, flavor, default_fill)))
+        return ret;
+
+    return 0;
+}
+
+/* Use the vara functions to read some data from an open test
+ * file. Expect only fill data. */
+int putget_read_vara_fill(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                          int default_fill, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* Read the data we wrote. */
+    if ((ret = PIOc_get_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_text(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_short(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_int(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_float(ncid, varid[4], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_double(ncid, varid[5], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN/2; x++)
+    {
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != (default_fill ? NC_FILL_BYTE : byte_fill_value))
+                return ERR_WRONG;
+            if (text_array_in[x][y] != (default_fill ? NC_FILL_CHAR : char_fill_value))
+                return ERR_WRONG;
+            if (short_array_in[x][y] != (default_fill ? NC_FILL_SHORT : short_fill_value))
+                return ERR_WRONG;
+            if (int_array_in[x][y] != (default_fill ? NC_FILL_INT : int_fill_value))
+                return ERR_WRONG;
+            if (float_array_in[x][y] != (default_fill ? NC_FILL_FLOAT : float_fill_value))
+                return ERR_WRONG;
+            if (double_array_in[x][y] != (default_fill ? NC_FILL_DOUBLE : double_fill_value))
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara_uchar(ncid, varid[6], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ushort(ncid, varid[7], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_uint(ncid, varid[8], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_longlong(ncid, varid[9], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ulonglong(ncid, varid[10], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN/2; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != (default_fill ? NC_FILL_UBYTE : ubyte_fill_value))
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != (default_fill ? NC_FILL_USHORT : ushort_fill_value))
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != (default_fill ? NC_FILL_UINT : uint_fill_value))
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != (default_fill ? NC_FILL_INT64 : int64_fill_value))
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != (default_fill ? NC_FILL_UINT64 : uint64_fill_value))
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Create a test file for the tests to write data to and check by
+ * reading it back. In this function we create the file, define the
+ * dims and vars, and pass back the ncid.
+ *
+ * @param iosysid the IO system ID.
+ * @param try the number of the test run, 0 for var, 1 for var1, 2 for
+ * vara, 3 for vars.
+ * @param flavor the PIO IO type.
+ * @param dim_len array of length NDIM of the dimension lengths.
+ * @param varid array of varids for the variables in the test file.
+ * @param filename the name of the test file to create.
+ * @param default_fill true if default fill values are to be used.
+ * @param ncidp pointer that gets the ncid of the created file.
+ * @returns 0 for success, error code otherwise.
+ */
+int create_putget_file(int iosysid, int flavor, int *dim_len, int *varid, const char *filename,
+                       int default_fill, int *ncidp)
+{
+    int dimids[NDIM];        /* The dimension IDs. */
+    int num_vars = NUM_CLASSIC_TYPES;
+    int xtype[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                    PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING};
+    int ncid;
+    int ret;
+
+    /* Create the netCDF output file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor, filename, PIO_CLOBBER)))
+        return ret;
+
+    /* Turn on fill mode. */
+    int old_mode;
+    if ((ret = PIOc_set_fill(ncid, NC_FILL, &old_mode)))
+        return ret;
+    printf("old_mode = %d\n", old_mode);
+    if (old_mode != NC_NOFILL)
+        return ERR_WRONG;
+
+    /* Define netCDF dimensions and variable. */
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            return ret;
+
+    /* For netcdf-4, there are extra types. */
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+        num_vars = NUM_NETCDF_TYPES;
+
+    /* Define variables. */
+    for (int v = 0; v < num_vars; v++)
+    {
+        char var_name[PIO_MAX_NAME + 1];
+        snprintf(var_name, PIO_MAX_NAME, "%s_%d", VAR_NAME, xtype[v]);
+        printf("defining var %s\n", var_name);
+        if ((ret = PIOc_def_var(ncid, var_name, xtype[v], NDIM, dimids, &varid[v])))
+            return ret;
+    }
+
+    /* Maybe set fill values. */
+    if (!default_fill)
+    {
+        if ((ret = PIOc_def_var_fill(ncid, varid[0], NC_FILL, &byte_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[1], NC_FILL, &char_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[2], NC_FILL, &short_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[3], NC_FILL, &int_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[4], NC_FILL, &float_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[5], NC_FILL, &double_fill_value)))
+            return ret;
+        if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+        {
+            if ((ret = PIOc_def_var_fill(ncid, varid[6], NC_FILL, &ubyte_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[7], NC_FILL, &ushort_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[8], NC_FILL, &uint_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[9], NC_FILL, &int64_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[10], NC_FILL, &uint64_fill_value)))
+                return ret;
+        }
+    }
+
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Pass back the ncid. */
+    *ncidp = ncid;
+
+    return 0;
+}
+
+/* Test data fill values.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the vara functions to write, and then read a
+ * subset of the variable data from the test file. The rest of the
+ * data are fill values.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_fill(int iosysid, int num_flavors, int *flavor, int my_rank,
+              MPI_Comm test_comm)
+{
+    int dim_len[NDIM] = {NUM_TIMESTEPS, X_DIM_LEN, Y_DIM_LEN};
+
+    printf("running tests for %d flavors\n", num_flavors);
+
+    /* Test with and without default fill values. */
+    for (int default_fill = 0; default_fill < 2; default_fill++)
+    {
+        /* Use PIO to create the example file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++)
+        {
+            char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+            char iotype_name[PIO_MAX_NAME + 1];
+            int ncid;
+            int varid[NUM_NETCDF_TYPES];
+            int ret;    /* Return code. */
+
+            /* Create a filename. */
+            if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+                return ret;
+            snprintf(filename, PIO_MAX_NAME, "%s_default_fill_%d_%s.nc", TEST_NAME, default_fill, iotype_name);
+
+            /* Create test file with dims and vars defined. */
+            printf("%d creating test file %s for flavor = %d...\n", my_rank, filename, flavor[fmt]);
+            if ((ret = create_putget_file(iosysid, flavor[fmt], dim_len, varid, filename,
+                                          default_fill, &ncid)))
+                return ret;
+            printf("created file %s\n", filename);
+
+            /* Write some data. */
+            PIO_Offset start[NDIM] = {1, 0, 0};
+            PIO_Offset count[NDIM] = {1, X_DIM_LEN/2, Y_DIM_LEN};
+
+            printf("writing data to %s\n", filename);
+            /* Use the no-type vara functions to write some data. */
+            if ((ret = putget_write_vara(ncid, varid, start, count, flavor[fmt])))
+                return ret;
+
+            printf("wrote data to %s\n", filename);
+            /* Make sure all data are written (pnetcdf needs this). */
+            if ((ret = PIOc_sync(ncid)))
+                return ret;
+
+            /* Use the vara functions to read some data. */
+            if ((ret = putget_read_vara(ncid, varid, start, count, default_fill, flavor[fmt])))
+                return ret;
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+            /* Access to read it. */
+            printf("about to try to open file %s\n", filename);
+            if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_WRITE)))
+                ERR(ret);
+
+            /* Use the vara functions to read some data. */
+            if ((ret = putget_read_vara(ncid, varid, start, count, default_fill, flavor[fmt])))
+                return ret;
+
+            /* Use the vara functions to read some data which are just fill values. */
+            start[0] = 0;
+            if ((ret = putget_read_vara_fill(ncid, varid, start, count, default_fill, flavor[fmt])))
+                return ret;
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+        } /* next flavor */
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test fill mode in files. Create a file with no data, and make sure
+ * fill mode is working correctly for all iotypes.
+ *
+ * This function creates a file with 1 dim (of length 1), 1 var. Fill
+ * mode is turned on for the file. No data is written to the
+ * file. Then the file is closed and re-opened, and the data value
+ * checked.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_fill_mode(int iosysid, int num_flavors, int *flavor, int my_rank,
+                   MPI_Comm test_comm, int async)
+{
+#define NUM_TYPES_TO_TEST 2
+    int xtype[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT};
+
+    printf("test_fill_mode async = %d\n", async);
+
+    /* Test with and without default fill values. */
+    for (int default_fill = 0; default_fill < 2; default_fill++)
+    {
+        /* Test with and without extra var that does get data. */
+        for (int extra_var = 0; extra_var < 2; extra_var++)
+        {
+            /* Use PIO to create the example file in each of the four
+             * available ways. */
+            for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+                {
+                    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+                    char iotype_name[PIO_MAX_NAME + 1];
+                    int ncid;
+                    int dimid;
+                    int varid;
+                    int varid2;
+                    void *fillvalue;
+                    PIO_Offset type_size;
+                    int default_int_fill = NC_FILL_INT;
+                    float default_float_fill = NC_FILL_FLOAT;
+                    void *extra_data;
+                    int extra_data_int = TEST_VAL_42;
+                    float extra_data_float = TEST_VAL_42;
+                    int ret;    /* Return code. */
+
+                    /* Create a filename. */
+                    if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+                        return ret;
+                    snprintf(filename, PIO_MAX_NAME, "%s_fill_mode_async_%d_default_fill_%d_extra_var_%d_%s.nc",
+                             TEST_NAME, async, default_fill, extra_var, iotype_name);
+
+                    /* Create the test file. */
+                    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                        return ret;
+
+                    /* Turn on fill mode. */
+                    if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+                        return ret;
+
+                    /* Define a dimension. */
+                    if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)))
+                        return ret;
+
+                    /* Define a variable. */
+                    if ((ret = PIOc_def_var(ncid, VAR_NAME, xtype[t], 1, &dimid, &varid)))
+                        return ret;
+
+                    /* Do we want an extra variable? */
+                    if (extra_var)
+                        if ((ret = PIOc_def_var(ncid, VAR_NAME_2, xtype[t], 1, &dimid, &varid2)))
+                            return ret;
+
+                    /* Find the size of our type. */
+                    if ((ret = PIOc_inq_type(ncid, xtype[t], NULL, &type_size)))
+                        return ret;
+                    if ((xtype[t] == PIO_INT || xtype[t] == PIO_FLOAT) && type_size != 4)
+                        return ERR_WRONG;
+
+                    /* Determine fill value and extra data, depending on type. */
+                    switch (xtype[t])
+                    {
+                    case PIO_INT:
+                        fillvalue = (default_fill ? &default_int_fill: &int_fill_value);
+                        extra_data = &extra_data_int;
+                        break;
+                    case PIO_FLOAT:
+                        fillvalue = (default_fill ? &default_float_fill: &float_fill_value);
+                        extra_data = &extra_data_float;
+                        break;
+                    default:
+                        return ERR_WRONG;
+                    }
+                
+                    /* If not using a default fill value, set one. */
+                    if (!default_fill)
+                        if ((ret = PIOc_def_var_fill(ncid, varid, NC_FILL, fillvalue)))
+                            return ret;
+
+                    /* End define mode. */
+                    if ((ret = PIOc_enddef(ncid)))
+                        return ret;
+
+                    /* If there is an extra variable, write data to it. */
+                    if (extra_var)
+                        if ((ret = PIOc_put_var(ncid, varid2, extra_data)))
+                            return ret;
+                    
+                    /* Close the netCDF file. */
+                    if ((ret = PIOc_closefile(ncid)))
+                        ERR(ret);
+
+                    /* Access to read it. */
+                    printf("about to try to open file %s\n", filename);
+                    if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, PIO_WRITE)))
+                        ERR(ret);
+
+                    /* Check the fill value. */
+                    void *fillvalue_in;
+                    int fill_mode_in;
+
+                    if (!(fillvalue_in = malloc(type_size)))
+                        return PIO_ENOMEM;
+                    if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode_in, fillvalue_in)))
+                        return ret;
+                    if (fill_mode_in != NC_FILL)
+                        return ERR_WRONG;
+                    if (memcmp(fillvalue_in, fillvalue, type_size))
+                        return ERR_WRONG;
+                    free(fillvalue_in);
+
+                    /* Allocate space to read one element of data. */
+                    void *data_in;
+                    if (!(data_in = malloc(type_size)))
+                        return PIO_ENOMEM;
+
+                    /* Use the vara functions to read 1 datum from
+                     * the var we didn't write to. We should get a
+                     * fill value. */
+                    if ((ret = PIOc_get_var(ncid, varid, data_in)))
+                        return ret;
+                    if (memcmp(data_in, fillvalue, type_size))
+                        return ERR_WRONG;
+
+                    /* Use the vara functions to read 1 datum from the
+                     * var we did write to. We should get the value we
+                     * wrote. */
+                    if (extra_var)
+                    {
+                        if ((ret = PIOc_get_var(ncid, varid2, data_in)))
+                            return ret;
+                        if (memcmp(data_in, extra_data, type_size))
+                            return ERR_WRONG;
+                    }
+
+                    /* Free memory. */
+                    free(data_in);
+
+                    /* Close the netCDF file. */
+                    if ((ret = PIOc_closefile(ncid)))
+                        ERR(ret);
+                } /* next type */
+            } /* with or without extra var */
+        } /* next flavor */
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* Test that empty vars get fill values when fill mode is on. */
+    if ((ret = test_fill_mode(iosysid, num_flavors, flavor, my_rank, test_comm,
+                              async)))
+        ERR(ret);
+
+    /* Test read/write stuff. */
+    if ((ret = test_fill(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Run Tests for NetCDF-4 Functions. */
+int main(int argc, char **argv)
+{
+    int ret;
+
+    /* Initialize data arrays with sample data. */
+    init_arrays();
+
+    /* Change the 5th arg to 3 to turn on logging. */
+    if ((ret = run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0,
+                             TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS)))
+        return ret;
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc_putget.c b/src/externals/pio2/tests/cunit/test_pioc_putget.c
new file mode 100644
index 00000000000..02e9177be54
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc_putget.c
@@ -0,0 +1,2082 @@
+/*
+ * Tests for PIO data reading and writing routines.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc_putget"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 40
+#define Y_DIM_LEN 40
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 1
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* The name of the attribute in the netCDF output files. */
+#define ATT_NAME "bar"
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Values for some netcdf-4 settings. */
+#define VAR_CACHE_SIZE (1024 * 1024)
+#define VAR_CACHE_NELEMS 10
+#define VAR_CACHE_PREEMPTION 0.5
+
+/* Number of NetCDF classic types. */
+#define NUM_CLASSIC_TYPES 6
+
+/* Number of NetCDF-4 types. */
+#define NUM_NETCDF4_TYPES 12
+
+/* Length of the attributes for each type. */
+#define ATT_LEN 3
+
+/* Text att (must be two chars long to fit in ATT_LEN with a NULL). */
+#define TEXT_ATT_VALUE "hi"
+
+/* Names for each type attribute. */
+#define TEXT_ATT_NAME "text_att"
+#define SCHAR_ATT_NAME "schar_att"
+#define SHORT_ATT_NAME "short_att"
+#define INT_ATT_NAME "int_att"
+#define LONG_ATT_NAME "long_att"
+#define FLOAT_ATT_NAME "float_att"
+#define DOUBLE_ATT_NAME "double_att"
+#define UCHAR_ATT_NAME "uchar_att"
+#define USHORT_ATT_NAME "ushort_att"
+#define UINT_ATT_NAME "uint_att"
+#define INT64_ATT_NAME "int64_att"
+#define UINT64_ATT_NAME "unit64_att"
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* Length of chunksizes to use in netCDF-4 files. */
+PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
+
+/* Some sample data values to write. */
+char text[] = "hi";
+char char_data = 2;
+signed char byte_data = -42;
+short short_data = -300;
+int int_data = -10000;
+long int long_data = -20000;
+float float_data = -42.42;
+double double_data = -420000000000.5;
+unsigned char ubyte_data = 43;
+unsigned short ushort_data = 666;
+unsigned int uint_data = 666666;
+long long int64_data = -99999999999;
+unsigned long long uint64_data = 99999999999;
+
+char text_array[X_DIM_LEN][Y_DIM_LEN];
+char char_array[X_DIM_LEN][Y_DIM_LEN];
+signed char byte_array[X_DIM_LEN][Y_DIM_LEN];
+short short_array[X_DIM_LEN][Y_DIM_LEN];
+int int_array[X_DIM_LEN][Y_DIM_LEN];
+long int long_array[X_DIM_LEN][Y_DIM_LEN];
+float float_array[X_DIM_LEN][Y_DIM_LEN];
+double double_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned char ubyte_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned short ushort_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned int uint_array[X_DIM_LEN][Y_DIM_LEN];
+long long int64_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned long long uint64_array[X_DIM_LEN][Y_DIM_LEN];
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+#define DIM_LEN 4
+
+/* Fill up the data arrays with some values. */
+void init_arrays()
+{
+    for (int x = 0; x < X_DIM_LEN; x++)
+    {
+        strcpy(text_array[x], text);
+        for (int y = 0; y < Y_DIM_LEN; y++)
+        {
+            char_array[x][y] = char_data;
+            byte_array[x][y] = byte_data;
+            short_array[x][y] = short_data;
+            int_array[x][y] = int_data;
+            long_array[x][y] = int_data;
+            float_array[x][y] = float_data;
+            double_array[x][y] = double_data;
+            ubyte_array[x][y] = ubyte_data;
+            ushort_array[x][y] = ushort_data;
+            uint_array[x][y] = uint_data;
+            int64_array[x][y] = int64_data;
+            uint64_array[x][y] = uint64_data;
+        }
+    }
+}
+
+/**
+ * Test the attribute conversions.
+ *
+ * @param ncid the ncid of the open test file.
+ * @param flavor the iotype of the test file.
+ * @param expected array of int, one per type. Expected return value.
+ * @param expected_data array of values we expect to find in attributes.
+ * @return 0 for success, error code otherwise.
+ */
+int test_att_conv_byte(int ncid, int flavor, char *name, int *expected, long long *expected_data)
+{
+    signed char byte_array_in[ATT_LEN];
+    short short_array_in[ATT_LEN];
+    unsigned char ubyte_array_in[ATT_LEN];
+    int int_array_in[ATT_LEN];
+    long long_array_in[ATT_LEN];
+    float float_array_in[ATT_LEN];
+    double double_array_in[ATT_LEN];
+    unsigned short ushort_array_in[ATT_LEN];
+    unsigned int uint_array_in[ATT_LEN];
+    long long int64_array_in[ATT_LEN];
+    unsigned long long uint64_array_in[ATT_LEN];
+
+    /* Read the att and check results. */
+    printf("expecting %d got %d\n", expected[PIO_BYTE], PIOc_get_att_schar(ncid, NC_GLOBAL, name, byte_array_in));
+
+    if (expected[PIO_BYTE] != PIOc_get_att_schar(ncid, NC_GLOBAL, name, byte_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_BYTE] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (byte_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_SHORT] != PIOc_get_att_short(ncid, NC_GLOBAL, name, short_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_SHORT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (short_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_INT] != PIOc_get_att_int(ncid, NC_GLOBAL, name, int_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_INT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (int_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_INT] != PIOc_get_att_long(ncid, NC_GLOBAL, name, long_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_INT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (long_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_FLOAT] != PIOc_get_att_float(ncid, NC_GLOBAL, name, float_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_FLOAT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (float_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_DOUBLE] != PIOc_get_att_double(ncid, NC_GLOBAL, name, double_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_DOUBLE] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (double_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((expected[PIO_UBYTE] != PIOc_get_att_uchar(ncid, NC_GLOBAL, name, ubyte_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_UBYTE] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (ubyte_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_USHORT] != PIOc_get_att_ushort(ncid, NC_GLOBAL, name, ushort_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_USHORT] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (ushort_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_UINT] != PIOc_get_att_uint(ncid, NC_GLOBAL, name, uint_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_UINT] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (uint_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_INT64] != PIOc_get_att_longlong(ncid, NC_GLOBAL, name, int64_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_INT64] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (int64_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_UINT64] != PIOc_get_att_ulonglong(ncid, NC_GLOBAL, name, uint64_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_UINT64] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (uint64_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test the attribute conversions.
+ *
+ * @param ncid the ncid of the open test file.
+ * @param flavor the iotype of the test file.
+ * @param expected array of int, one per type. Expected return value.
+ * @param expected_data array of values we expect to find in attributes.
+ * @return 0 for success, error code otherwise.
+ */
+int test_att_conv_int64(int ncid, int flavor, char *name, int *expected, long long *expected_data)
+{
+    float float_array_in[ATT_LEN];
+    double double_array_in[ATT_LEN];
+    unsigned char ubyte_array_in[ATT_LEN];
+    unsigned short ushort_array_in[ATT_LEN];
+    unsigned int uint_array_in[ATT_LEN];
+    long long int64_array_in[ATT_LEN];
+
+    /* Read the att and check results. */
+    if (expected[PIO_FLOAT] != PIOc_get_att_float(ncid, NC_GLOBAL, name, float_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_DOUBLE] != PIOc_get_att_double(ncid, NC_GLOBAL, name, double_array_in))
+        return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((expected[PIO_UBYTE] != PIOc_get_att_uchar(ncid, NC_GLOBAL, name, ubyte_array_in)))
+            return ERR_WRONG;
+        if ((expected[PIO_USHORT] != PIOc_get_att_ushort(ncid, NC_GLOBAL, name, ushort_array_in)))
+            return ERR_WRONG;
+        if ((expected[PIO_UINT] != PIOc_get_att_uint(ncid, NC_GLOBAL, name, uint_array_in)))
+            return ERR_WRONG;
+
+        if ((expected[PIO_INT64] != PIOc_get_att_longlong(ncid, NC_GLOBAL, name, int64_array_in)))
+            return ERR_WRONG;
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test attribute read/write operations with NC_BYTE data.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the var/var1/vars/vars functions to write, and
+ * then read data from the test file.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_atts_byte(int iosysid, int num_flavors, int *flavor, int my_rank,
+                   MPI_Comm test_comm)
+{
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char iotype_name[PIO_MAX_NAME + 1];
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        int ncid;
+        int ret;    /* Return code. */
+
+        /* Create test file with dims and vars defined. */
+        printf("%d creating test file for flavor = %d...\n", my_rank, flavor[fmt]);
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_att_byte_%s.nc", TEST_NAME, iotype_name);
+
+        /* Create the netCDF output file. */
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            return ret;
+
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, SCHAR_ATT_NAME, PIO_BYTE, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, SHORT_ATT_NAME, PIO_SHORT, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, INT_ATT_NAME, PIO_INT, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, FLOAT_ATT_NAME, PIO_FLOAT, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, PIO_DOUBLE, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, UCHAR_ATT_NAME, PIO_UBYTE, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, USHORT_ATT_NAME, PIO_USHORT, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, UINT_ATT_NAME, PIO_UINT, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+            if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, INT64_ATT_NAME, PIO_INT64, ATT_LEN, (signed char *)byte_array)))
+                return ret;
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, UINT64_ATT_NAME, PIO_UINT64, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+        }
+        if ((ret = PIOc_enddef(ncid)))
+            return ret;
+
+        /* Test the attribute conversions. */
+        long long expected_data[ATT_LEN];
+        for (int x = 0; x < ATT_LEN; x++)
+            expected_data[x] = byte_array[x][0];
+
+        int schar_expected[NUM_NETCDF_TYPES + 1] = {0, 0, PIO_ERANGE, 0, 0, 0, 0, PIO_ERANGE, PIO_ERANGE, PIO_ERANGE,
+                                                    0, PIO_ERANGE, PIO_ERANGE};
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SCHAR_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SHORT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], FLOAT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], DOUBLE_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT64_ATT_NAME, schar_expected, expected_data)))
+                ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Test the attribute conversions. */
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SCHAR_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SHORT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], FLOAT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], DOUBLE_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT64_ATT_NAME, schar_expected, expected_data)))
+                ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+    } /* next flavor */
+
+    return PIO_NOERR;
+}
+
+/* Test attribute read/write operations with NC_INT64 data.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the var/var1/vars/vars functions to write, and
+ * then read data from the test file.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_atts_int64(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm)
+{
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char iotype_name[PIO_MAX_NAME + 1];
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        int ncid;
+        int ret;    /* Return code. */
+
+        /* Only do these tests on netCDF-4 files. */
+        if (flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+            continue;
+
+        /* Create test file with dims and vars defined. */
+        printf("%d creating test file for flavor = %d...\n", my_rank, flavor[fmt]);
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_att_int64_%s.nc", TEST_NAME, iotype_name);
+
+        /* Create the netCDF output file. */
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            return ret;
+
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, SCHAR_ATT_NAME, PIO_BYTE, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, SHORT_ATT_NAME, PIO_SHORT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, INT_ATT_NAME, PIO_INT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if ((ret = PIOc_put_att_longlong(ncid, NC_GLOBAL, FLOAT_ATT_NAME, PIO_FLOAT, ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_att_longlong(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, PIO_DOUBLE, ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, UCHAR_ATT_NAME, PIO_UBYTE, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, USHORT_ATT_NAME, PIO_USHORT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, UINT_ATT_NAME, PIO_UINT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if ((ret = PIOc_put_att_longlong(ncid, NC_GLOBAL, INT64_ATT_NAME, PIO_INT64, ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, UINT64_ATT_NAME, PIO_UINT64, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if ((ret = PIOc_enddef(ncid)))
+            return ret;
+
+        /* Test the attribute conversions. */
+        long long expected_data[ATT_LEN];
+        for (int x = 0; x < ATT_LEN; x++)
+            expected_data[x] = int64_array[x][0];
+
+        int int64_expected[NUM_NETCDF_TYPES + 1] = {0, PIO_ERANGE, PIO_ERANGE, PIO_ERANGE, PIO_ERANGE, 0, 0, 0, 0, 0,
+                                                    0, PIO_ERANGE, PIO_ERANGE};
+        if ((ret = test_att_conv_int64(ncid, flavor[fmt], SCHAR_ATT_NAME, int64_expected, expected_data)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        if ((ret = test_att_conv_int64(ncid, flavor[fmt], SCHAR_ATT_NAME, int64_expected, expected_data)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+    } /* next flavor */
+
+    return PIO_NOERR;
+}
+
+/* Use the var1 functions to write some data to an open test file. */
+int putget_write_var1(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_put_var1_text(ncid, varid[1], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_put_var1_text(ncid + TEST_VAL_42, varid[1], index, text) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_put_var1_text(ncid, varid[1] + TEST_VAL_42, index, text) != PIO_ENOTVAR)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_put_var1_text(ncid, varid[1], index, text)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_schar(ncid, varid[0], index, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_short(ncid, varid[2], index, &short_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_int(ncid, varid[3], index, &int_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_long(ncid, varid[4], index, &long_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_float(ncid, varid[5], index, &float_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_double(ncid, varid[6], index, &double_data)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var1_uchar(ncid, varid[7], index, &ubyte_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_ushort(ncid, varid[8], index, &ushort_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_uint(ncid, varid[9], index, &uint_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_longlong(ncid, varid[10], index, &int64_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_ulonglong(ncid, varid[11], index, &uint64_data)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the no type versions of the var1 functions to write some data
+ * to an open test file. */
+int putget_write_var1_nt(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_put_var1(ncid, varid[1], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_put_var1(ncid + TEST_VAL_42, varid[1], index, text) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_put_var1(ncid, varid[1] + TEST_VAL_42, index, text) != PIO_ENOTVAR)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_put_var1(ncid, varid[0], index, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[1], index, text)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[2], index, &short_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[3], index, &int_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[4], index, &int_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[5], index, &float_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[6], index, &double_data)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var1(ncid, varid[7], index, &ubyte_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[8], index, &ushort_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[9], index, &uint_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[10], index, &int64_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[11], index, &uint64_data)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the var functions to write some data to an open test file. */
+int putget_write_var(int ncid, int *varid, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_var_text(ncid, varid[1], (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_schar(ncid, varid[0], (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_short(ncid, varid[2], (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_int(ncid, varid[3], (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_long(ncid, varid[4], (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_float(ncid, varid[5], (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_double(ncid, varid[6], (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var_uchar(ncid, varid[7], (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_var_ushort(ncid, varid[8], (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_var_uint(ncid, varid[9], (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_var_longlong(ncid, varid[10], (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_var_ulonglong(ncid, varid[11], (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the var functions to write some data to an open test file. */
+int putget_write_var_nt(int ncid, int *varid, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[1], (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[0], (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[2], (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[3], (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[4], (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[5], (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[6], (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var(ncid, varid[7], (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[8], (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[9], (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[10], (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[11], (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the att functions to write some data to attributes in an open
+ * test file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_write_atts(int ncid, int *varid, int flavor)
+{
+    int ret;
+
+    /* Test some invalid parameters. (Type is irrelevant here.) */
+    if (PIOc_put_att_schar(ncid + 1, varid[0], SCHAR_ATT_NAME, PIO_BYTE,
+                           ATT_LEN, (signed char *)byte_array) != PIO_EBADID)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_put_att_text(ncid, varid[0], TEXT_ATT_NAME, ATT_LEN,
+                                 TEXT_ATT_VALUE)))
+        return ret;
+
+    /* Use put_att() for the schar. */
+    if ((ret = PIOc_put_att(ncid, varid[0], SCHAR_ATT_NAME, PIO_BYTE,
+                            ATT_LEN, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_text(ncid, varid[1], TEXT_ATT_NAME, ATT_LEN,
+                                 TEXT_ATT_VALUE)))
+        return ret;
+
+    if ((ret = PIOc_put_att_short(ncid, varid[2], SHORT_ATT_NAME, PIO_SHORT,
+                                  ATT_LEN, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_int(ncid, varid[3], INT_ATT_NAME, PIO_INT,
+                                  ATT_LEN, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_long(ncid, varid[4], LONG_ATT_NAME, PIO_INT,
+                                 ATT_LEN, (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_float(ncid, varid[5], FLOAT_ATT_NAME, PIO_FLOAT,
+                                  ATT_LEN, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_double(ncid, varid[6], DOUBLE_ATT_NAME, PIO_DOUBLE,
+                                  ATT_LEN, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, PIO_UBYTE,
+                                      ATT_LEN, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_ushort(ncid, varid[8], USHORT_ATT_NAME, PIO_SHORT,
+                                       ATT_LEN, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_att_uint(ncid, varid[9], UINT_ATT_NAME, PIO_UINT,
+                                     ATT_LEN, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_att_longlong(ncid, varid[10], INT64_ATT_NAME, PIO_INT64,
+                                         ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_att_ulonglong(ncid, varid[11], UINT64_ATT_NAME, PIO_UINT64,
+                                          ATT_LEN, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the att functions to read some attributes from an open test
+ * file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_read_att(int ncid, int *varid, int flavor)
+{
+    char text_in[ATT_LEN];
+    signed char byte_array_in[ATT_LEN];
+    short short_array_in[ATT_LEN];
+    unsigned char ubyte_array_in[ATT_LEN];
+    int int_array_in[ATT_LEN];
+    long int long_array_in[ATT_LEN];
+    float float_array_in[ATT_LEN];
+    double double_array_in[ATT_LEN];
+    unsigned short ushort_array_in[ATT_LEN];
+    unsigned int uint_array_in[ATT_LEN];
+    long long int64_array_in[ATT_LEN];
+    unsigned long long uint64_array_in[ATT_LEN];
+    int x;
+    int ret;
+
+    if ((ret = PIOc_get_att_schar(ncid, varid[0], SCHAR_ATT_NAME, byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_text(ncid, varid[1], TEXT_ATT_NAME, text_in)))
+        return ret;
+    if ((ret = PIOc_get_att_short(ncid, varid[2], SHORT_ATT_NAME, short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_int(ncid, varid[3], INT_ATT_NAME, int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_long(ncid, varid[4], LONG_ATT_NAME, long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_float(ncid, varid[5], FLOAT_ATT_NAME, float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_double(ncid, varid[6], DOUBLE_ATT_NAME, double_array_in)))
+        return ret;
+    for (x = 0; x < ATT_LEN; x++)
+    {
+        if (strncmp(text_in, TEXT_ATT_VALUE, ATT_LEN))
+            return ERR_WRONG;
+        if (byte_array_in[x] != byte_array[x][0])
+            return ERR_WRONG;
+        if (short_array_in[x] != short_array[x][0])
+            return ERR_WRONG;
+        if (int_array_in[x] != int_array[x][0])
+            return ERR_WRONG;
+        if (long_array_in[x] != long_array[x][0])
+            return ERR_WRONG;
+        if (float_array_in[x] != float_array[x][0])
+            return ERR_WRONG;
+        if (double_array_in[x] != double_array[x][0])
+            return ERR_WRONG;
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_ushort(ncid, varid[8], USHORT_ATT_NAME, ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_uint(ncid, varid[9], UINT_ATT_NAME, uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_longlong(ncid, varid[10], INT64_ATT_NAME, int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_ulonglong(ncid, varid[11], UINT64_ATT_NAME, uint64_array_in)))
+            return ret;
+        for (x = 0; x < ATT_LEN; x++)
+        {
+            if (ubyte_array_in[x] != ubyte_array[x][0])
+                return ERR_WRONG;
+            if (ushort_array_in[x] != ushort_array[x][0])
+                return ERR_WRONG;
+            if (uint_array_in[x] != uint_array[x][0])
+                return ERR_WRONG;
+            if (int64_array_in[x] != int64_array[x][0])
+                return ERR_WRONG;
+            if (uint64_array_in[x] != uint64_array[x][0])
+                return ERR_WRONG;
+        }
+    }
+
+    return 0;
+}
+
+/* Use the vara functions to write some data to an open test file. */
+int putget_write_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                      int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara_text(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_int(ncid, varid[3], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_long(ncid, varid[4], start, count, (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_float(ncid, varid[5], start, count, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_double(ncid, varid[6], start, count, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vara_uchar(ncid, varid[7], start, count, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ushort(ncid, varid[8], start, count, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_uint(ncid, varid[9], start, count, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_longlong(ncid, varid[10], start, count, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ulonglong(ncid, varid[11], start, count, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the no-type vara functions to write some data to an open test file. */
+int putget_write_vara_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                         int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[0], start, count, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[2], start, count, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[3], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[4], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[5], start, count, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[6], start, count, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vara(ncid, varid[7], start, count, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[8], start, count, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[9], start, count, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[10], start, count, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[11], start, count, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the vars functions to write some data to an open test file. */
+int putget_write_vars(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                      PIO_Offset *stride, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara_text(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_schar(ncid, varid[0], start, count, stride, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_short(ncid, varid[2], start, count, stride, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_int(ncid, varid[3], start, count, stride, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_long(ncid, varid[4], start, count, stride, (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_float(ncid, varid[5], start, count, stride, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_double(ncid, varid[6], start, count, stride, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vars_uchar(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_ushort(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_uint(ncid, varid[9], start, count, stride, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_longlong(ncid, varid[10], start, count, stride, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_ulonglong(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the no-type vars functions to write some data to an open test file. */
+int putget_write_vars_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                         PIO_Offset *stride, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[0], start, count, stride, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[2], start, count, stride, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[3], start, count, stride, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[4], start, count, stride, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[5], start, count, stride, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[6], start, count, stride, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vars(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[9], start, count, stride, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[10], start, count, stride, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the var1 functions to read some data from an open test file. */
+int putget_read_var1(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    signed char byte_data_in;
+    char text_data_in[PIO_MAX_NAME];
+    short short_data_in;
+    unsigned char ubyte_data_in;
+    int int_data_in;
+    long int long_data_in;
+    float float_data_in;
+    double double_data_in;
+    unsigned short ushort_data_in;
+    unsigned int uint_data_in;
+    long long int64_data_in;
+    unsigned long long uint64_data_in;
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_get_var1_schar(ncid + TEST_VAL_42, varid[0], index, &byte_data_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_get_var1_schar(ncid, varid[0] + TEST_VAL_42, index, &byte_data_in) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_get_var1_schar(ncid, varid[0], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Get and check the data. */
+    if ((ret = PIOc_get_var1_schar(ncid, varid[0], index, &byte_data_in)))
+        return ret;
+    if (byte_data_in != byte_data)
+        return ERR_WRONG;
+
+    memset(text_data_in, 0, sizeof(text_data_in));
+    if ((ret = PIOc_get_var1_text(ncid, varid[1], index, text_data_in)))
+        return ret;
+    printf("text_data_in = %s\n", text_data_in);
+    if (strncmp(text_data_in, "h", 1))
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_short(ncid, varid[2], index, &short_data_in)))
+        return ret;
+    if (short_data_in != short_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_int(ncid, varid[3], index, &int_data_in)))
+        return ret;
+    if (int_data_in != int_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_long(ncid, varid[4], index, &long_data_in)))
+        return ret;
+    if (long_data_in != long_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_float(ncid, varid[5], index, &float_data_in)))
+        return ret;
+    if (float_data_in != float_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_double(ncid, varid[6], index, &double_data_in)))
+        return ret;
+    if (double_data_in != double_data)
+        return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var1_uchar(ncid, varid[7], index, &ubyte_data_in)))
+            return ret;
+        if (ubyte_data_in != ubyte_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_ushort(ncid, varid[8], index, &ushort_data_in)))
+            return ret;
+        if (ushort_data_in != ushort_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_uint(ncid, varid[9], index, &uint_data_in)))
+            return ret;
+        if (uint_data_in != uint_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_longlong(ncid, varid[10], index, &int64_data_in)))
+            return ret;
+        if (int64_data_in != int64_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_ulonglong(ncid, varid[11], index, &uint64_data_in)))
+            return ret;
+        if (uint64_data_in != uint64_data)
+            return ERR_WRONG;
+    }
+
+    return 0;
+}
+
+/* Use the var1 functions to read some data from an open test file. */
+int putget_read_var1_nt(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    signed char byte_data_in;
+    char text_data_in[PIO_MAX_NAME];
+    short short_data_in;
+    unsigned char ubyte_data_in;
+    int int_data_in;
+    float float_data_in;
+    double double_data_in;
+    unsigned short ushort_data_in;
+    unsigned int uint_data_in;
+    long long int64_data_in;
+    unsigned long long uint64_data_in;
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_get_var1(ncid + TEST_VAL_42, varid[0], index, &byte_data_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_get_var1(ncid, varid[0] + TEST_VAL_42, index, &byte_data_in) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_get_var1(ncid, varid[0], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Get and check the data. */
+    if ((ret = PIOc_get_var1(ncid, varid[0], index, &byte_data_in)))
+        return ret;
+    if (byte_data_in != byte_data)
+        return ERR_WRONG;
+
+    memset(text_data_in, 0, sizeof(text_data_in));
+    if ((ret = PIOc_get_var1(ncid, varid[1], index, text_data_in)))
+        return ret;
+    printf("text_data_in = %s\n", text_data_in);
+    if (strncmp(text_data_in, "h", 1))
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[2], index, &short_data_in)))
+        return ret;
+    if (short_data_in != short_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[3], index, &int_data_in)))
+        return ret;
+    if (int_data_in != int_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[4], index, &int_data_in)))
+        return ret;
+    if (int_data_in != int_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[5], index, &float_data_in)))
+        return ret;
+    if (float_data_in != float_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[6], index, &double_data_in)))
+        return ret;
+    if (double_data_in != double_data)
+        return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var1(ncid, varid[7], index, &ubyte_data_in)))
+            return ret;
+        if (ubyte_data_in != ubyte_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[8], index, &ushort_data_in)))
+            return ret;
+        if (ushort_data_in != ushort_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[9], index, &uint_data_in)))
+            return ret;
+        if (uint_data_in != uint_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[10], index, &int64_data_in)))
+            return ret;
+        if (int64_data_in != int64_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[11], index, &uint64_data_in)))
+            return ret;
+        if (uint64_data_in != uint64_data)
+            return ERR_WRONG;
+    }
+
+    return 0;
+}
+
+/* Use the var functions to read some data from an open test file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param unlim non-zero if unlimited dimension is in use.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+*/
+int putget_read_var(int ncid, int *varid, int unlim, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long int long_array_in[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* When using the unlimited dimension, no data are wrtten by the
+     * put_var_TYPE() functions, since the length of the unlimited
+     * dimension is still 0. */
+    if (unlim)
+    {
+        return 0;
+    }
+
+    if ((ret = PIOc_get_var_schar(ncid, varid[0], (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_text(ncid, varid[1], (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_short(ncid, varid[2], (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_int(ncid, varid[3], (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_long(ncid, varid[4], (long int *)long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_float(ncid, varid[5], (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_double(ncid, varid[6], (double *)double_array_in)))
+        return ret;
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (long_array_in[x][y] != long_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var_uchar(ncid, varid[7], (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_ushort(ncid, varid[8], (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_uint(ncid, varid[9], (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_longlong(ncid, varid[10], (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_ulonglong(ncid, varid[11], (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the no-type var functions to read some data from an open test file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param unlim non-zero if unlimited dimension is in use.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+*/
+int putget_read_var_nt(int ncid, int *varid, int unlim, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in2[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* When using the unlimited dimension, no data are wrtten by the
+     * put_var_TYPE() functions, since the length of the unlimited
+     * dimension is still 0. */
+    if (unlim)
+    {
+        return 0;
+    }
+
+    if ((ret = PIOc_get_var(ncid, varid[0], (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[1], (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[2], (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[3], (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[4], (int *)int_array_in2)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[5], (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[6], (double *)double_array_in)))
+        return ret;
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in2[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var(ncid, varid[7], (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[8], (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[9], (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[10], (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[11], (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the vara functions to read some data from an open test file. */
+int putget_read_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                     int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long int long_array_in[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_text(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_short(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_int(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_long(ncid, varid[4], start, count, (long int *)long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_float(ncid, varid[5], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_double(ncid, varid[6], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (long_array_in[x][y] != long_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara_uchar(ncid, varid[7], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vara_ushort(ncid, varid[8], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_uint(ncid, varid[9], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_longlong(ncid, varid[10], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ulonglong(ncid, varid[11], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the vars functions to read some data from an open test file. */
+int putget_read_vars(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                     PIO_Offset *stride, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long int long_array_in[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vars_schar(ncid, varid[0], start, count, stride, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_text(ncid, varid[1], start, count, stride, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_short(ncid, varid[2], start, count, stride, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_int(ncid, varid[3], start, count, stride, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_long(ncid, varid[4], start, count, stride, (long int *)long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_float(ncid, varid[5], start, count, stride, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_double(ncid, varid[6], start, count, stride, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+            return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (long_array_in[x][y] != long_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vars_uchar(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vars_ushort(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars_uint(ncid, varid[9], start, count, stride, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars_longlong(ncid, varid[10], start, count, stride, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars_ulonglong(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the no-type vara functions to read some data from an open test file. */
+int putget_read_vara_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                        int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in2[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vara(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[4], start, count, (int *)int_array_in2)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[5], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[6], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in2[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara(ncid, varid[7], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vara(ncid, varid[8], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara(ncid, varid[9], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara(ncid, varid[10], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara(ncid, varid[11], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the vars functions to read some data from an open test file. */
+int putget_read_vars_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                        PIO_Offset *stride, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in2[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vars(ncid, varid[0], start, count, stride, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[1], start, count, stride, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[2], start, count, stride, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[3], start, count, stride, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[4], start, count, stride, (int *)int_array_in2)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[5], start, count, stride, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[6], start, count, stride, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+            return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in2[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vars(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vars(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars(ncid, varid[9], start, count, stride, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars(ncid, varid[10], start, count, stride, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Create a test file for the putget tests to write data to and check
+ * by reading it back. In this function we create the file, define the
+ * dims and vars, and pass back the ncid.
+ *
+ * @param iosysid the IO system ID.
+ * @param try the number of the test run, 0 for var, 1 for var1, 2 for
+ * vara, 3 for vars.
+ * @param unlim non-zero if unlimited dimension should be used.
+ * @param flavor the PIO IO type.
+ * @param dim_len array of length NDIM of the dimension lengths.
+ * @param varid array of varids for the variables in the test file.
+ * @param filename the name of the test file to create.
+ * @param ncidp pointer that gets the ncid of the created file.
+ * @returns 0 for success, error code otherwise.
+ */
+int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_len,
+                       int *varid, const char *filename, int *ncidp)
+{
+    int dimids[NDIM];        /* The dimension IDs. */
+    int num_vars = NUM_CLASSIC_TYPES + 1;
+    int xtype[NUM_NETCDF4_TYPES + 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_LONG_INTERNAL,
+                                        PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64,
+                                        PIO_UINT64, PIO_STRING};
+    int ncid;
+    int ret;
+
+    /* This should not work. */
+    if (PIOc_createfile(iosysid + TEST_VAL_42, &ncid, &flavor, filename, PIO_CLOBBER) != PIO_EBADID)
+        return ERR_WRONG;
+
+    /* Create the netCDF output file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor, filename, PIO_CLOBBER)))
+        return ret;
+
+    /* Are we using unlimited dimension? */
+    if (!unlim)
+        dim_len[0] = NUM_TIMESTEPS;
+
+    printf("filename = %s\n", filename);
+    /* Define netCDF dimensions and variable. */
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            return ret;
+
+    /* For netcdf-4, there are extra types. */
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+        num_vars = NUM_NETCDF4_TYPES + 1;
+
+    printf("filename = %s\n", filename);
+    /* Define variables. */
+    for (int v = 0; v < num_vars; v++)
+    {
+        char var_name[PIO_MAX_NAME + 1];
+        snprintf(var_name, PIO_MAX_NAME, "%s_%d", VAR_NAME, xtype[v]);
+        printf("defining var %s\n", var_name);
+        /*nc_type my_type = xtype[v] == PIO_LONG_INTERNAL ? PIO_INT : xtype[v];*/
+        nc_type my_type;
+        if (xtype[v] == PIO_LONG_INTERNAL)
+            my_type = PIO_INT;
+        else
+            my_type = xtype[v];
+        if ((ret = PIOc_def_var(ncid, var_name, my_type, NDIM, dimids, &varid[v])))
+            return ret;
+    }
+
+    printf("filename = %s\n", filename);
+    /* For the first access, also test attributes. */
+    if (access == 0)
+        if ((ret = test_write_atts(ncid, varid, flavor)))
+            return ret;
+
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Pass back the ncid. */
+    *ncidp = ncid;
+
+    return 0;
+}
+
+/** Check the contents of the test file. */
+int check_file(int access, int ncid, int *varid, int flavor, PIO_Offset *index, PIO_Offset *start,
+               PIO_Offset *count, PIO_Offset *stride, int unlim)
+{
+    int ret;
+    
+    switch (access)
+    {
+    case 0:
+        /* Use the att functions to read some data. */
+        if ((ret = test_read_att(ncid, varid, flavor)))
+            return ret;
+
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_var(ncid, varid, unlim, flavor)))
+            return ret;
+        break;
+
+    case 1:
+        /* Use the var1 functions to read some data. */
+        if ((ret = putget_read_var1(ncid, varid, index, flavor)))
+            return ret;
+        break;
+
+    case 2:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vara(ncid, varid, start, count, flavor)))
+            return ret;
+        break;
+
+    case 3:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vars(ncid, varid, start, count, stride, flavor)))
+            return ret;
+        break;
+
+    case 4:
+        /* Use the var1 functions to read some data. */
+        if ((ret = putget_read_var_nt(ncid, varid, unlim, flavor)))
+            return ret;
+        break;
+
+    case 5:
+        /* Use the var1 functions to read some data. */
+        if ((ret = putget_read_var1_nt(ncid, varid, index, flavor)))
+            return ret;
+        break;
+
+    case 6:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vara_nt(ncid, varid, start, count, flavor)))
+            return ret;
+        break;
+
+    case 7:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vars_nt(ncid, varid, start, count, stride, flavor)))
+            return ret;
+        break;
+
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test data read/write operations.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the var/var1/vars/vars functions to write, and
+ * then read data from the test file.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
+                MPI_Comm test_comm)
+{
+    int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+#define NUM_ACCESS 8
+    for (int unlim = 0; unlim < 2; unlim++)
+        for (int access = 0; access < NUM_ACCESS; access++)
+        {
+            /* Use PIO to create the example file in each of the four
+             * available ways. */
+            for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+                char iotype_name[PIO_MAX_NAME + 1];
+                int ncid;
+                int varid[NUM_NETCDF4_TYPES + 1];
+                int ret;    /* Return code. */
+
+                /* Create a filename. */
+                if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+                    return ret;
+                snprintf(filename, PIO_MAX_NAME, "%s_putget_access_%d_unlim_%d_%s.nc", TEST_NAME,
+                         access, unlim, iotype_name);
+
+                /* Create test file with dims and vars defined. */
+                printf("%d Access %d creating test file %s for flavor = %d...\n", my_rank, access,
+                       filename, flavor[fmt]);
+                if ((ret = create_putget_file(iosysid, access, unlim, flavor[fmt], dim_len, varid,
+                                              filename, &ncid)))
+                    return ret;
+                printf("created file %s\n", filename);
+
+                /* Write some data. */
+                PIO_Offset index[NDIM] = {0, 0, 0};
+                PIO_Offset start[NDIM] = {0, 0, 0};
+                PIO_Offset count[NDIM] = {1, X_DIM_LEN, Y_DIM_LEN};
+                PIO_Offset stride[NDIM] = {1, 1, 1};
+
+                switch (access)
+                {
+                case 0:
+                    printf("%d Access %d writing data with var functions for flavor = %d...\n",
+                           my_rank, access, flavor[fmt]);
+                    /* Use the var functions to write some data. */
+                    if ((ret = putget_write_var(ncid, varid, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 1:
+                    /* Use the var1 functions to write some data. */
+                    if ((ret = putget_write_var1(ncid, varid, index, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 2:
+                    /* Use the vara functions to write some data. */
+                    if ((ret = putget_write_vara(ncid, varid, start, count, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 3:
+                    /* Use the vara functions to write some data. */
+                    if ((ret = putget_write_vars(ncid, varid, start, count, stride, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 4:
+                    /* Use the no-type var functions to write some data. */
+                    if ((ret = putget_write_var_nt(ncid, varid, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 5:
+                    /* Use the no-type var1 functions to write some data. */
+                    if ((ret = putget_write_var1_nt(ncid, varid, index, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 6:
+                    /* Use the no-type vara functions to write some data. */
+                    if ((ret = putget_write_vara_nt(ncid, varid, start, count, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 7:
+                    /* Use the no-type vars functions to write some data. */
+                    if ((ret = putget_write_vars_nt(ncid, varid, start, count, stride, flavor[fmt])))
+                        return ret;
+                    break;
+
+                }
+
+                /* Check for bad input handling. */
+                if (PIOc_sync(ncid + TEST_VAL_42) != PIO_EBADID)
+                    return ret;
+
+                /* Make sure all data are written (pnetcdf needs this). */
+                if ((ret = PIOc_sync(ncid)))
+                    return ret;
+
+                /* Check contents of the file. */
+                if ((ret = check_file(access, ncid, varid, flavor[fmt], index, start, count, stride, unlim)))
+                    return ret;
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* /\* Access to read it. *\/ */
+                printf("about to try to open file %s\n", filename);
+                if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+                    ERR(ret);
+
+                /* Check contents of the file. */
+                if ((ret = check_file(access, ncid, varid, flavor[fmt], index, start, count, stride, unlim)))
+                    return ret;
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+            } /* next flavor */
+        } /* next access */
+
+    return PIO_NOERR;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* Test attribute stuff. */
+    printf("%d Testing attributes with NC_BYTE data, async = %d\n", my_rank, async);
+    if ((ret = test_atts_byte(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        return ret;
+
+    printf("%d Testing attributes with NC_INT64 data, async = %d\n", my_rank, async);
+    if ((ret = test_atts_int64(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        return ret;
+
+    /* Test read/write stuff. */
+    printf("%d Testing putget. async = %d\n", my_rank, async);
+    if ((ret = test_putget(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run Tests for NetCDF-4 Functions. */
+int main(int argc, char **argv)
+{
+    /* Initialize data arrays with sample data. */
+    init_arrays();
+
+    return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0,
+                         TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc_unlim.c b/src/externals/pio2/tests/cunit/test_pioc_unlim.c
new file mode 100644
index 00000000000..3b4a4df347c
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc_unlim.c
@@ -0,0 +1,188 @@
+/*
+ * Tests for PIO Functions. In this test we use a simple 3D variable,
+ * with an unlimited dimension. The data will have two timesteps, and
+ * 4x4 elements each timestep.
+ *
+ * Ed Hartnett, 2/14/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc_unlim"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "var_3D_with_unlim"
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* Create the decomposition to divide the 3-dimensional sample data
+ * between the 4 tasks.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim1_len the length of the dimension.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len,
+                         int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int ret;
+
+    /* How many data elements per task? */
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM - 1, &dim_len[1], elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/* Create the test file. */
+int create_test_file(int iosysid, int ioid, int iotype, int my_rank, int *ncid, int *varid)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; /* Length of the dimensions in the sample data. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ret;       /* Return code. */
+
+    /* Create the filename. */
+    sprintf(filename, "%s_iotype_%d.nc", TEST_NAME, iotype);
+    
+    /* Create the netCDF output file. */
+    printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+           iotype);
+    if ((ret = PIOc_createfile(iosysid, ncid, &iotype, filename, PIO_CLOBBER)))
+        ERR(ret);
+    
+    /* Define netCDF dimensions and variable. */
+    printf("rank: %d Defining netCDF metadata...\n", my_rank);
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(*ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            ERR(ret);
+    
+    /* Define a variable. */
+    if ((ret = PIOc_def_var(*ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, varid)))
+        ERR(ret);
+    
+    /* End define mode. */
+    if ((ret = PIOc_enddef(*ncid)))
+        ERR(ret);
+    
+    return PIO_NOERR;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int ioid;
+    int ncid;
+    int varid;
+    int my_test_size;
+    char filename[NC_MAX_NAME + 1];
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+    
+    /* This will be our file name for writing out decompositions. */
+    sprintf(filename, "decomp_%d.txt", my_rank);
+
+    if (!async)
+    {
+        printf("%d Testing darray. async = %d\n", my_rank, async);
+        
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition(my_test_size, my_rank, iosysid, X_DIM_LEN, &ioid)))
+            return ret;
+
+        /* Use PIO to create the example file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++)
+        {
+            if ((ret = create_test_file(iosysid, ioid, flavor[fmt], my_rank, &ncid, &varid)))
+                return ret;
+            if ((ret = PIOc_setframe(ncid, varid, 0)))
+                return ret;
+            if ((ret = PIOc_advanceframe(ncid, varid)))
+                return ret;
+
+            /* Look at the internals to check that the frame commands
+             * worked. */
+            file_desc_t *file;            
+            if ((ret = pio_get_file(ncid, &file)))
+                return ret;
+            if (file->varlist[varid].record != 1)
+                return ERR_WRONG;
+            
+            if ((PIOc_closefile(ncid)))
+                return ret;
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run Tests for NetCDF-4 Functions. */
+int main(int argc, char **argv)
+{
+    /* Change the 5th arg to 3 to turn on logging. */
+    return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 3,
+                         TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
+}
diff --git a/src/externals/pio2/tests/cunit/test_shared.c b/src/externals/pio2/tests/cunit/test_shared.c
new file mode 100644
index 00000000000..3d6eb9cd1ff
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_shared.c
@@ -0,0 +1,199 @@
+/*
+ * Common test code for PIO C tests.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* Test with async.
+ *
+ * @param my_rank rank of the task.
+ * @param nprocs the size of the communicator.
+ * @param num_flavors the number of PIO IO types that will be tested.
+ * @param flavors array of the PIO IO types that will be tested.
+ * @param test_comm communicator with all test tasks.
+ * @returns 0 for success error code otherwise.
+ */
+int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm,
+                int component_count, int num_io_procs, int target_ntasks, char *test_name)
+{
+    int iosysid[component_count];  /* The ID for the parallel I/O system. */
+    int num_procs[component_count + 1]; /* Num procs in each component. */
+    MPI_Comm io_comm;              /* Will get a duplicate of IO communicator. */
+    MPI_Comm comp_comm[component_count]; /* Will get duplicates of computation communicators. */
+    int mpierr;  /* Return code from MPI functions. */
+    int ret;     /* Return code. */
+
+    num_procs[0] = 1;
+    num_procs[1] = target_ntasks - 1;
+
+    /* Is the current process a computation task? */
+    int comp_task = my_rank < num_io_procs ? 0 : 1;
+    printf("%d comp_task = %d\n", my_rank, comp_task);
+
+    /* Initialize the IO system. */
+    if ((ret = PIOc_Init_Async(test_comm, num_io_procs, NULL, component_count,
+                               num_procs, NULL, &io_comm, comp_comm, iosysid)))
+        ERR(ERR_INIT);
+    for (int c = 0; c < component_count; c++)
+        printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
+
+    /* All the netCDF calls are only executed on the computation
+     * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+     * and when the do, they should go straight to finalize. */
+    if (comp_task)
+    {
+        for (int c = 0; c < component_count; c++)
+        {
+            printf("%d Running tests...\n", my_rank);
+            if ((ret = test_all(iosysid[c], num_flavors, flavor, my_rank, comp_comm[0], 1)))
+                return ret;
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+            printf("%d %s Freeing PIO resources\n", my_rank, test_name);
+            if ((ret = PIOc_finalize(iosysid[c])))
+                ERR(ret);
+            printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, test_name,
+                   iosysid[c]);
+            if ((mpierr = MPI_Comm_free(&comp_comm[c])))
+                MPIERR(mpierr);
+        }
+    }
+    else
+    {
+        if ((mpierr = MPI_Comm_free(&io_comm)))
+            MPIERR(mpierr);
+    } /* endif comp_task */
+
+    return PIO_NOERR;
+}
+
+/* Test without async.
+ *
+ * @param my_rank rank of the task.
+ * @param num_flavors the number of PIO IO types that will be tested.
+ * @param flavors array of the PIO IO types that will be tested.
+ * @param test_comm communicator with all test tasks.
+ * @returns 0 for success error code otherwise.
+ */
+int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm, int target_ntasks,
+                   int x_dim_len, int y_dim_len)
+{
+    int niotasks;    /* Number of processors that will do IO. */
+    int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+    int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+    PIO_Offset elements_per_pe; /* Array index per processing unit. */
+    int iosysid;  /* The ID for the parallel I/O system. */
+    int ioid;     /* The I/O description ID. */
+    PIO_Offset *compdof; /* The decomposition mapping. */
+    int slice_dimlen[2];
+    int ret;      /* Return code. */
+
+    slice_dimlen[0] = x_dim_len;
+    slice_dimlen[1] = y_dim_len;
+
+    /* keep things simple - 1 iotask per MPI process */
+    niotasks = target_ntasks;
+
+    /* Initialize the PIO IO system. This specifies how
+     * many and which processors are involved in I/O. */
+    if ((ret = PIOc_Init_Intracomm(test_comm, niotasks, ioproc_stride,
+                                   ioproc_start, PIO_REARR_SUBSET, &iosysid)))
+        return ret;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    elements_per_pe = x_dim_len * y_dim_len / target_ntasks;
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition...\n", my_rank);
+    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                               compdof, &ioid, NULL, NULL, NULL)))
+        return ret;
+    free(compdof);
+
+    /* Run tests. */
+    printf("%d Running tests...\n", my_rank);
+    if ((ret = test_all(iosysid, num_flavors, flavor, my_rank, test_comm, 0)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    printf("%d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+
+    /* Finalize PIO system. */
+    if ((ret = PIOc_finalize(iosysid)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/*
+ * Run the test code, executing the test_all() function without async,
+ * and then with async.
+ *
+ * @param argc argument count passed into main.
+ * @param argv argument array passed into main.
+ * @param min_ntasks the min number of tasks this code need to run.
+ * @param max_ntasks the code will run on this many tasks, extras will
+ * be ignored.
+ * @param log_level the level to set the logger at.
+ * @param test_name the name of the test being run.
+ * @param dim_len array of size 3 with the dimension sizes.
+ * @param component_count number of computation components.
+ * @param num_io_procs number of IO components.
+ * @returns 0 for success, error code otherwise.
+ */
+int run_test_main(int argc, char **argv, int min_ntasks, int max_ntasks,
+                  int log_level, char *test_name, int *dim_len, int component_count,
+                  int num_io_procs)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, min_ntasks,
+                              max_ntasks, log_level, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < max_ntasks)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        /* Run tests without async feature. */
+        if ((ret = test_no_async2(my_rank, num_flavors, flavor, test_comm, max_ntasks,
+                                  dim_len[1], dim_len[2])))
+            return ret;
+
+        /* Run tests with async. */
+        if ((ret = test_async2(my_rank, num_flavors, flavor, test_comm, component_count,
+                               num_io_procs, max_ntasks, test_name)))
+            return ret;
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, test_name);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, test_name);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_spmd.c b/src/externals/pio2/tests/cunit/test_spmd.c
new file mode 100644
index 00000000000..b4947137004
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_spmd.c
@@ -0,0 +1,535 @@
+/*
+ * This program tests some internal functions in the library.
+ *
+ * Jim Edwards
+ * Ed Hartnett, 11/23/16
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 1
+
+/* The name of this test. */
+#define TEST_NAME "test_spmd"
+
+/* Number of test cases in inner loop of test. */
+#define NUM_TEST_CASES 5
+
+#define TEST_MAX_GATHER_BLOCK_SIZE 32
+
+/* Test MPI_Alltoallw by having processor i send different amounts of
+ * data to each processor.  The first test sends i items to processor
+ * i from all processors. */
+int run_spmd_tests(MPI_Comm test_comm)
+{
+    int my_rank;  /* 0-based rank in test_comm. */
+    int ntasks;   /* Number of tasks in test_comm. */
+    int num_elem; /* Number of elements in buffers. */
+    int type_size; /* Size in bytes of an element. */
+    int mpierr;   /* Return value from MPI calls. */
+    int ret;      /* Return value. */
+
+    /* Learn rank and size. */
+    if ((mpierr = MPI_Comm_size(test_comm, &ntasks)))
+        MPIERR(mpierr);
+    if ((mpierr = MPI_Comm_rank(test_comm, &my_rank)))
+        MPIERR(mpierr);
+
+    /* Determine size of buffers. */
+    num_elem = ntasks;
+
+    int sbuf[ntasks];    /* The send buffer. */
+    int rbuf[ntasks];    /* The receive buffer. */
+    int sendcounts[ntasks]; /* Number of elements of data being sent from each task. */
+    int recvcounts[ntasks]; /* Number of elements of data being sent from each task. */
+    int sdispls[ntasks]; /* Displacements for sending data. */
+    int rdispls[ntasks]; /* Displacements for receiving data. */
+    MPI_Datatype sendtypes[ntasks]; /* MPI types of data being sent. */
+    MPI_Datatype recvtypes[ntasks]; /* MPI types of data being received. */
+
+    /* Load up the send buffer. */
+    for (int i = 0; i < num_elem; i++)
+        sbuf[i] = my_rank;
+
+    /* Load up the receive buffer to make debugging easier. */
+    for (int i = 0; i < num_elem; i++)
+        rbuf[i] = -999;
+
+    /* Get the size of the int type for MPI. (Should always be 4.) */
+    if ((mpierr = MPI_Type_size(MPI_INT, &type_size)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    assert(type_size == sizeof(int));
+
+    /* Initialize the arrays. */
+    for (int i = 0; i < ntasks; i++)
+    {
+        sendcounts[i] = 1;
+        sdispls[i] = 0;
+        sendtypes[i] = MPI_INT;
+        recvcounts[i] = 1;
+        rdispls[i] = i * type_size;
+        recvtypes[i] = MPI_INT;
+    }
+
+    /* Perform tests for different values of msg_cnt. (BTW it hangs
+     * with msg_cnt = 1!). */
+    for (int msg_cnt = 0; msg_cnt < TARGET_NTASKS; msg_cnt = msg_cnt ? msg_cnt * 2 : 4)
+    {
+        if (!my_rank)
+            printf("message count %d\n",msg_cnt);
+
+        for (int itest = 0; itest < NUM_TEST_CASES; itest++)
+        {
+            bool hs = false;
+            bool isend = false;
+
+            /* Wait for all tasks. */
+            MPI_Barrier(test_comm);
+
+            /* Print results. */
+            if (!my_rank)
+                for (int e = 0; e < num_elem; e++)
+                    printf("sbuf[%d] = %d\n", e, sbuf[e]);
+
+            /* Set the parameters different for each test case. */
+            if (itest == 1)
+            {
+                hs = true;
+                isend = true;
+            }
+            else if (itest == 2)
+            {
+                hs = false;
+                isend = true;
+            }
+            else if (itest == 3)
+            {
+                hs = false;
+                isend = false;
+            }
+            else if (itest == 4)
+            {
+                hs = true;
+                isend = false;
+            }
+
+            /* Run the swapm function. */
+            if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts,
+                                 rdispls, recvtypes, test_comm, hs, isend, msg_cnt)))
+                return ret;
+
+            /* Print results. */
+            /* MPI_Barrier(test_comm); */
+            /* for (int e = 0; e < num_elem; e++) */
+            /*     printf("%d sbuf[%d] = %d\n", my_rank, e, sbuf[e]); */
+            /* MPI_Barrier(test_comm); */
+            /* for (int e = 0; e < num_elem; e++) */
+            /*     printf("%d rbuf[%d] = %d\n", my_rank, e, rbuf[e]); */
+
+            /* Check that rbuf has 0, 1, ..., ntasks-1. */
+            for (int e = 0; e < num_elem; e++)
+                if (((int *)rbuf)[e] != e)
+                    return ERR_WRONG;
+        }
+    }
+
+    return 0;
+}
+
+/* Test some of the functions in the file pioc_sc.c. 
+ *
+ * @param test_comm the MPI communicator that the test code is running on. 
+ * @returns 0 for success, error code otherwise.
+ */
+int run_sc_tests(MPI_Comm test_comm)
+{
+#define SC_ARRAY_LEN 3
+    int my_rank;  /* 0-based rank in test_comm. */
+    int ntasks;   /* Number of tasks in test_comm. */
+    int mpierr;   /* Return value from MPI calls. */
+    int array1[SC_ARRAY_LEN] = {7, 42, 14};
+    int array2[SC_ARRAY_LEN] = {2, 3, 7};
+    int array3[SC_ARRAY_LEN] = {90, 180, 270};
+    int array4[SC_ARRAY_LEN] = {1, 180, 270};
+
+    /* Learn rank and size. */
+    if ((mpierr = MPI_Comm_size(test_comm, &ntasks)))
+        MPIERR(mpierr);
+    if ((mpierr = MPI_Comm_rank(test_comm, &my_rank)))
+        MPIERR(mpierr);
+
+    /* Test the gcd() function. */
+    if (gcd(0, 2) != 2)
+        return ERR_WRONG;
+    if (gcd(2, 2) != 2)
+        return ERR_WRONG;
+    if (gcd(42, 2) != 2)
+        return ERR_WRONG;
+
+    /* Test the long long version. */
+    if (lgcd(0, 2) != 2)
+        return ERR_WRONG;
+    if (lgcd(2, 2) != 2)
+        return ERR_WRONG;
+    if (lgcd(42, 2) != 2)
+        return ERR_WRONG;
+
+    /* Test the gcd_array() function. */
+    if (gcd_array(SC_ARRAY_LEN, array1) != 7)
+        return ERR_WRONG;
+    if (gcd_array(SC_ARRAY_LEN, array2) != 1)
+        return ERR_WRONG;
+    if (gcd_array(SC_ARRAY_LEN, array3) != 90)
+        return ERR_WRONG;
+    if (gcd_array(SC_ARRAY_LEN, array4) != 1)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* This test code was recovered from main() in pioc_sc.c. */
+int test_CalcStartandCount()
+{
+    int ndims = 2;
+    int gdims[2] = {31, 777602};
+    int num_io_procs = 24;
+    bool converged = false;
+    PIO_Offset start[ndims], kount[ndims];
+    int iorank, numaiotasks = 0;
+    long int tpsize = 0;
+    long int psize;
+    long int pgdims = 1;
+    int scnt;
+
+    for (int i = 0; i < ndims; i++)
+        pgdims *= gdims[i];
+
+    while (!converged)
+    {
+        for (iorank = 0; iorank < num_io_procs; iorank++)
+        {
+            numaiotasks = CalcStartandCount(PIO_DOUBLE, ndims, gdims, num_io_procs, iorank,
+                                            start, kount);
+            if (iorank < numaiotasks)
+                printf("iorank %d start %lld %lld count %lld %lld\n", iorank, start[0],
+                       start[1], kount[0], kount[1]);
+
+            if (numaiotasks < 0)
+                return numaiotasks;
+
+            psize = 1;
+            scnt = 0;
+            for (int i = 0; i < ndims; i++)
+            {
+                psize *= kount[i];
+                scnt += kount[i];
+            }
+            tpsize += psize;
+        }
+
+        if (tpsize == pgdims)
+            converged = true;
+        else
+        {
+            printf("Failed to converge %ld %ld %d\n", tpsize, pgdims, num_io_procs);
+            tpsize = 0;
+            num_io_procs--;
+        }
+    }
+
+    return 0;
+}
+
+/* Tesst some list stuff. */
+int test_lists()
+{
+    file_desc_t *fdesc;
+    
+    /* Test that bad input is correctly rejected. */
+    if (pio_delete_iodesc_from_list(42) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pio_delete_iosystem_from_list(42) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pio_delete_file_from_list(42) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pio_get_file(42, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (pio_get_file(42, &fdesc) != PIO_EBADID)
+        return ERR_WRONG;
+    return 0;
+}
+
+/* Test some of the rearranger utility functions. */
+int test_rearranger_opts1()
+{
+    rearr_comm_fc_opt_t *ro1;
+    rearr_comm_fc_opt_t *ro2;
+    rearr_comm_fc_opt_t *ro3;
+
+    if (!(ro1 = calloc(1, sizeof(rearr_comm_fc_opt_t))))
+        return ERR_AWFUL;
+    if (!(ro2 = calloc(1, sizeof(rearr_comm_fc_opt_t))))
+        return ERR_AWFUL;
+    if (!(ro3 = calloc(1, sizeof(rearr_comm_fc_opt_t))))
+        return ERR_AWFUL;
+
+    /* This should not work. */
+    if (PIOc_set_rearr_opts(42, 1, 1, 0, 0, 0, 0, 0, 0) != PIO_EBADID)
+        return ERR_WRONG;
+
+    /* ro1 and ro2 are the same. */
+    if (!cmp_rearr_comm_fc_opts(ro1, ro2))
+        return ERR_WRONG;
+
+    /* Make ro3 different. */
+    ro3->enable_hs = 1;
+    if (cmp_rearr_comm_fc_opts(ro1, ro3))
+        return ERR_WRONG;
+    ro3->enable_hs = 0;
+    ro3->enable_isend = 1;
+    if (cmp_rearr_comm_fc_opts(ro1, ro3))
+        return ERR_WRONG;
+    ro3->enable_isend = 0;
+    ro3->max_pend_req = 1;
+    if (cmp_rearr_comm_fc_opts(ro1, ro3))
+        return ERR_WRONG;
+
+    /* Free resourses. */
+    free(ro1);
+    free(ro2);
+    free(ro3);
+    
+    return 0;
+}
+
+/* Test some of the rearranger utility functions. */
+int test_rearranger_opts2()
+{
+    iosystem_desc_t my_ios;
+    iosystem_desc_t *ios = &my_ios;
+
+    /* I'm not sure what the point of this function is... */
+    check_and_reset_rearr_opts(ios);
+    
+    return 0;
+}
+
+/* Test the compare_offsets() function. */
+int test_compare_offsets()
+{
+    mapsort m1, m2, m3;
+
+    m1.rfrom = 0;
+    m1.soffset = 0;
+    m1.iomap = 0;
+    m2.rfrom = 0;
+    m2.soffset = 0;
+    m2.iomap = 0;
+    m3.rfrom = 0;
+    m3.soffset = 0;
+    m3.iomap = 1;
+
+    /* Return 0 if either or both parameters are null. */
+    if (compare_offsets(NULL, &m2))
+        return ERR_WRONG;
+    if (compare_offsets(&m1, NULL))
+        return ERR_WRONG;
+    if (compare_offsets(NULL, NULL))
+        return ERR_WRONG;
+
+    /* m1 and m2 are the same. */
+    if (compare_offsets(&m1, &m2))
+        return ERR_WRONG;
+
+    /* m1 and m3 are different. */
+    if (compare_offsets(&m1, &m3) != -1)
+        return ERR_WRONG;
+    return 0;
+}
+
+/* Test the ceil2() and pair() functions. */
+int test_ceil2_pair()
+{
+    /* Test the ceil2() function. */
+    if (ceil2(1) != 1)
+        return ERR_WRONG;
+    if (ceil2(-100) != 1)
+        return ERR_WRONG;
+    if (ceil2(2) != 2)
+        return ERR_WRONG;
+    if (ceil2(3) != 4)
+        return ERR_WRONG;
+    if (ceil2(16) != 16)
+        return ERR_WRONG;
+    if (ceil2(17) != 32)
+        return ERR_WRONG;
+
+    /* Test the pair() function. */
+    if (pair(4, 0, 0) != 1)
+        return ERR_WRONG;
+    if (pair(4, 2, 2) != 1)
+        return ERR_WRONG;
+    
+    return 0;
+}
+
+/* Test the function that finds an MPI type to match a PIO type. */
+int test_find_mpi_type()
+{
+    MPI_Datatype mpi_type;
+    int ret;
+
+    /* This should not work. */
+    if (find_mpi_type(PIO_BYTE + 42, &mpi_type) != PIO_EBADTYPE)
+        return ERR_WRONG;
+
+    /* Try every atomic type. */
+    if ((ret = find_mpi_type(PIO_BYTE, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_BYTE)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_CHAR, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_CHAR)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_SHORT, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_SHORT)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_INT, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_INT)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_FLOAT, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_FLOAT)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_DOUBLE, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_DOUBLE)
+        return ERR_WRONG;
+
+#ifdef _NETCDF4
+    if ((ret = find_mpi_type(PIO_UBYTE, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED_CHAR)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_USHORT, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED_SHORT)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_UINT, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_INT64, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_LONG_LONG)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_UINT64, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED_LONG_LONG)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_STRING, &mpi_type)))
+        return ret;
+    if (mpi_type != MPI_CHAR)
+        return ERR_WRONG;
+
+#endif /* _NETCDF4 */
+    return PIO_NOERR;
+}
+
+int test_misc()
+{
+    wmulti_buffer wmb;
+
+    /* This should not work. */
+    if (flush_buffer(TEST_VAL_42, &wmb, 0) != PIO_EBADID)
+        return ERR_WRONG;
+    
+    return 0;
+}
+
+/* Run Tests for pio_spmd.c functions. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks;  /* Number of processors involved in current execution. */
+    int ret;     /* Return code. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              TARGET_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        printf("%d running tests for functions in pioc_sc.c\n", my_rank);
+        if ((ret = run_sc_tests(test_comm)))
+            return ret;
+
+        printf("%d running spmd test code\n", my_rank);
+        if ((ret = run_spmd_tests(test_comm)))
+            return ret;
+        
+        printf("%d running CalcStartandCount test code\n", my_rank);
+        if ((ret = test_CalcStartandCount()))
+            return ret;
+
+        printf("%d running list tests\n", my_rank);
+        if ((ret = test_lists()))
+            return ret;
+
+        printf("%d running rearranger opts tests 1\n", my_rank);
+        if ((ret = test_rearranger_opts1()))
+            return ret;
+
+        printf("%d running rearranger opts tests 2\n", my_rank);
+        if ((ret = test_rearranger_opts2()))
+            return ret;
+
+        printf("%d running compare_offsets tests\n", my_rank);
+        if ((ret = test_compare_offsets()))
+            return ret;
+
+        printf("%d running ceil2/pair tests\n", my_rank);
+        if ((ret = test_ceil2_pair()))
+            return ret;
+
+        printf("%d running find_mpi_type tests\n", my_rank);
+        if ((ret = test_find_mpi_type()))
+            return ret;
+
+        printf("%d running misc tests\n", my_rank);
+        if ((ret = test_misc()))
+            return ret;
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/general/CMakeLists.txt b/src/externals/pio2/tests/general/CMakeLists.txt
new file mode 100644
index 00000000000..b71382d3064
--- /dev/null
+++ b/src/externals/pio2/tests/general/CMakeLists.txt
@@ -0,0 +1,832 @@
+include (LibMPI)
+
+#==============================================================================
+#  GENERATE TARGET SOURCES
+#==============================================================================
+
+SET(GENERATED_SRCS pio_file_simple_tests.F90
+  pio_init_finalize.F90
+  pio_fail.F90
+  pio_file_fail.F90
+  ncdf_simple_tests.F90
+  ncdf_get_put.F90
+  ncdf_fail.F90
+  ncdf_inq.F90
+  pio_rearr.F90
+  pio_rearr_opts.F90
+  pio_rearr_opts2.F90
+  pio_decomp_tests.F90
+  pio_decomp_tests_1d.F90
+  pio_decomp_tests_2d.F90
+  pio_decomp_tests_3d.F90
+  pio_decomp_frame_tests.F90
+  pio_decomp_fillval.F90
+  pio_iosystem_tests.F90
+  pio_iosystem_tests2.F90
+  pio_iosystem_tests3.F90)
+
+foreach (SRC_FILE IN LISTS GENERATED_SRCS)
+  add_custom_command (
+    OUTPUT ${SRC_FILE}
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tf_f90gen.pl
+    --annotate-source
+    --out=${CMAKE_CURRENT_BINARY_DIR}/${SRC_FILE}
+    ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in)
+endforeach ()
+
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  add_definitions(-ffree-line-length-none)
+endif()
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
+endif ()
+
+#==============================================================================
+#  DEFINE THE TARGETS AND TESTS
+#==============================================================================
+
+# Test Timeout (4 min = 240 sec)
+set (DEFAULT_TEST_TIMEOUT 240)
+
+#===== pio_init_finalize =====
+add_executable (pio_init_finalize EXCLUDE_FROM_ALL
+  pio_init_finalize.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_init_finalize piof)
+add_dependencies (tests pio_init_finalize)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME init_finialize_1_proc
+    COMMAND pio_init_finalize)
+  set_tests_properties(init_finialize_1_proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME init_finialize_2_proc
+    COMMAND pio_init_finalize)
+  set_tests_properties(init_finialize_2_proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME init_finalize_2_proc_with_args
+    COMMAND pio_init_finalize --pio-tf-stride=2 --pio-tf-num-aggregators=2)
+  set_tests_properties(init_finalize_2_proc_with_args
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(init_finialize_1_proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(init_finialize_2_proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(init_finalize_2_proc_with_args
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+    ARGUMENTS --pio-tf-stride=2 --pio-tf-num-aggregators=2
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_file_simple_tests =====
+add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL
+  pio_file_simple_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_file_simple_tests piof)
+add_dependencies (tests pio_file_simple_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_file_simple_tests
+    COMMAND pio_file_simple_tests)
+  set_tests_properties(pio_file_simple_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_file_simple_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_file_fail =====
+add_executable (pio_file_fail EXCLUDE_FROM_ALL
+  pio_file_fail.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_file_fail piof)
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (pio_init_finalize
+    PRIVATE -ffree-line-length-none)
+endif()
+add_dependencies (tests pio_file_fail)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_file_fail
+    COMMAND pio_file_fail)
+  set_tests_properties(pio_file_fail
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_file_fail
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_fail
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_simple_tests =====
+add_executable (ncdf_simple_tests EXCLUDE_FROM_ALL
+  ncdf_simple_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_simple_tests piof)
+add_dependencies (tests ncdf_simple_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_simple_tests
+    COMMAND ncdf_simple_tests)
+  set_tests_properties(ncdf_simple_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_simple_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_simple_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_get_put =====
+add_executable (ncdf_get_put EXCLUDE_FROM_ALL
+  ncdf_get_put.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_get_put piof)
+add_dependencies (tests ncdf_get_put)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_get_put_1proc
+    COMMAND ncdf_get_put)
+  set_tests_properties(ncdf_get_put_1proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME ncdf_get_put_2proc
+    COMMAND ncdf_get_put)
+  set_tests_properties(ncdf_get_put_2proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_get_put_1proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_get_put
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(ncdf_get_put_2proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_get_put
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_fail =====
+add_executable (ncdf_fail EXCLUDE_FROM_ALL
+  ncdf_fail.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_fail piof)
+add_dependencies (tests ncdf_fail)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_fail
+    COMMAND ncdf_fail)
+  set_tests_properties(ncdf_fail
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_fail
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_fail
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_inq =====
+add_executable (ncdf_inq EXCLUDE_FROM_ALL
+  ncdf_inq.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_inq piof)
+add_dependencies (tests ncdf_inq)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_inq
+    COMMAND ncdf_inq)
+  set_tests_properties(ncdf_inq
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_inq
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_inq
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_rearr =====
+add_executable (pio_rearr EXCLUDE_FROM_ALL
+    pio_rearr.F90
+    ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_rearr piof)
+add_dependencies (tests pio_rearr)
+
+if (PIO_USE_MPISERIAL)
+    add_test(NAME pio_rearr
+        COMMAND pio_rearr)
+    set_tests_properties(pio_rearr
+        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+    add_mpi_test(pio_rearr
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr
+        NUMPROCS 4
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_rearr_opts =====
+add_executable (pio_rearr_opts EXCLUDE_FROM_ALL
+    pio_rearr_opts.F90
+    ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_rearr_opts piof)
+add_dependencies (tests pio_rearr_opts)
+
+if (PIO_USE_MPISERIAL)
+    add_test(NAME pio_rearr_opts
+        COMMAND pio_rearr_opts)
+    set_tests_properties(pio_rearr_opts
+        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+    add_mpi_test(pio_rearr_opts
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts
+        NUMPROCS 4
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_rearr_opts2 =====
+add_executable (pio_rearr_opts2 EXCLUDE_FROM_ALL
+    pio_rearr_opts2.F90
+    ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_rearr_opts2 piof)
+add_dependencies (tests pio_rearr_opts2)
+
+if (PIO_USE_MPISERIAL)
+    add_test(NAME pio_rearr_opts2
+        COMMAND pio_rearr_opts2)
+    set_tests_properties(pio_rearr_opts2
+        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+    add_mpi_test(pio_rearr_opts2_1p
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts2
+        NUMPROCS 1
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+    add_mpi_test(pio_rearr_opts2_3p
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts2
+        NUMPROCS 3
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+    add_mpi_test(pio_rearr_opts2_4p
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts2
+        NUMPROCS 4
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests =====
+add_executable (pio_decomp_tests EXCLUDE_FROM_ALL
+  pio_decomp_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests piof)
+add_dependencies (tests pio_decomp_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_1p
+    COMMAND pio_decomp_tests)
+  set_tests_properties(pio_decomp_tests_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2p
+    COMMAND pio_decomp_tests)
+  set_tests_properties(pio_decomp_tests_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3p
+    COMMAND pio_decomp_tests)
+  set_tests_properties(pio_decomp_tests_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_4p_1agg
+    COMMAND pio_decomp_tests --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_2agg
+    COMMAND pio_decomp_tests --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_3agg
+    COMMAND pio_decomp_tests --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_4p_1iop
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_2iop
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_3iop
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_4p_2iop_2str
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_2iop_1agg
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests_1d =====
+add_executable (pio_decomp_tests_1d EXCLUDE_FROM_ALL
+  pio_decomp_tests_1d.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests_1d piof)
+add_dependencies (tests pio_decomp_tests_1d)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_1d_1p
+    COMMAND pio_decomp_tests_1d)
+  set_tests_properties(pio_decomp_tests_1d_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_2p
+    COMMAND pio_decomp_tests_1d)
+  set_tests_properties(pio_decomp_tests_1d_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_3p
+    COMMAND pio_decomp_tests_1d)
+  set_tests_properties(pio_decomp_tests_1d_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_1d_4p_1agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_1d_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_2agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_1d_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_3agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_1d_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_1d_4p_1iop
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_1d_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_2iop
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_1d_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_3iop
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_1d_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_1d_4p_2iop_2str
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_1d_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_2iop_1agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_1d_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_1d_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_1d_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_1d_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_1d_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests_2d =====
+add_executable (pio_decomp_tests_2d EXCLUDE_FROM_ALL
+  pio_decomp_tests_2d.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests_2d piof)
+add_dependencies (tests pio_decomp_tests_2d)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_2d_1p
+    COMMAND pio_decomp_tests_2d)
+  set_tests_properties(pio_decomp_tests_2d_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_2p
+    COMMAND pio_decomp_tests_2d)
+  set_tests_properties(pio_decomp_tests_2d_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_3p
+    COMMAND pio_decomp_tests_2d)
+  set_tests_properties(pio_decomp_tests_2d_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_2d_4p_1agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_2d_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_2agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_2d_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_3agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_2d_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_2d_4p_1iop
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_2d_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_2iop
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_2d_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_3iop
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_2d_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_2d_4p_2iop_2str
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_2d_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_2iop_1agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_2d_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_2d_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_2d_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_2d_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_2d_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests_3d =====
+add_executable (pio_decomp_tests_3d EXCLUDE_FROM_ALL
+  pio_decomp_tests_3d.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests_3d piof)
+add_dependencies (tests pio_decomp_tests_3d)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_3d_1p
+    COMMAND pio_decomp_tests_3d)
+  set_tests_properties(pio_decomp_tests_3d_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_2p
+    COMMAND pio_decomp_tests_3d)
+  set_tests_properties(pio_decomp_tests_3d_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_3p
+    COMMAND pio_decomp_tests_3d)
+  set_tests_properties(pio_decomp_tests_3d_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_3d_4p_1agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_3d_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_2agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_3d_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_3agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_3d_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_3d_4p_1iop
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_3d_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_2iop
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_3d_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_3iop
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_3d_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_3d_4p_2iop_2str
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_3d_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_2iop_1agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_3d_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_3d_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_3d_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_3d_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_3d_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_frame_tests =====
+add_executable (pio_decomp_frame_tests EXCLUDE_FROM_ALL
+  pio_decomp_frame_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_frame_tests piof)
+add_dependencies (tests pio_decomp_frame_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_frame_tests
+    COMMAND pio_decomp_frame_tests)
+  set_tests_properties(pio_decomp_frame_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_frame_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_frame_tests
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_fillval =====
+add_executable (pio_decomp_fillval EXCLUDE_FROM_ALL
+  pio_decomp_fillval.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_fillval piof)
+add_dependencies (tests pio_decomp_fillval)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_fillval
+    COMMAND pio_decomp_fillval)
+  set_tests_properties(pio_decomp_fillval
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_fillval
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_fillval
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_iosystems_test =====
+add_executable (pio_iosystem_tests EXCLUDE_FROM_ALL
+  pio_iosystem_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_iosystem_tests piof)
+add_dependencies (tests pio_iosystem_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_iosystem_tests
+    COMMAND pio_iosystem_tests)
+  set_tests_properties(pio_iosystem_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_iosystem_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_iosystem_tests
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+
+#===== pio_iosystems_test2 =====
+add_executable (pio_iosystem_tests2 EXCLUDE_FROM_ALL
+  pio_iosystem_tests2.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_iosystem_tests2 piof)
+add_dependencies (tests pio_iosystem_tests2)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_iosystem_tests2
+    COMMAND pio_iosystem_tests2)
+  set_tests_properties(pio_iosystem_tests2
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_iosystem_tests2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_iosystem_tests2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_iosystems_test3 =====
+add_executable (pio_iosystem_tests3 EXCLUDE_FROM_ALL
+  pio_iosystem_tests3.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_iosystem_tests3 piof)
+add_dependencies (tests pio_iosystem_tests3)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_iosystem_tests3
+    COMMAND pio_iosystem_tests3)
+  set_tests_properties(pio_iosystem_tests3
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_iosystem_tests3
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_iosystem_tests3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
diff --git a/src/externals/pio2/tests/general/README.md b/src/externals/pio2/tests/general/README.md
new file mode 100644
index 00000000000..83539789881
--- /dev/null
+++ b/src/externals/pio2/tests/general/README.md
@@ -0,0 +1,142 @@
+# Using the testing framework
+ PIO uses a custom testing framework (`/tests/general/util/*`) to generate tests in the `/tests/general` directory. The testing framework provides,
+
+* Faster creation of tests : Easier creation of tests from simple stub source
+* Improved testcase code readability : Most of the boiler plate code is generated by the framework
+* Better error checking and reporting : User friendly parallel asserts
+* More testing with less code : Templated tests, auto generation of boiler plate PIO code
+* Easy integration with CTest 
+
+# Writing a test
+
+## Testcase format
+ All testcases recognized by the framework need to be subroutines with the following tags to indicate the start and end of the procedure,
+
+* `PIO_TF_AUTO_TEST_SUB_BEGIN` : This tag is used in the first line of the subroutine definition followed by the subroutine name. Consider this tag as the replacement of the `SUBROUTINE` keyword in Fortran
+* `PIO_TF_AUTO_TEST_SUB_END` : This tag marks the end of the subroutine, like the `"END SUBRTOUINE"` keyword in Fortran. This tag is followed by the subroutine name.
+
+ A simple hello world test program is shown below,
+
+    PIO_TF_AUTO_TEST_SUB_BEGIN hellow
+
+      print *, "Hello world"
+
+    PIO_TF_AUTO_TEST_SUB_END hellow
+
+ The testing framework converts this stub into a Fortran subroutine. The testing framework also adds a test driver that includes necessary code to initialize (and finalize) MPI, PIO and includes calls to all testcases tagged with the `PIO_TF_AUTO_TEST_SUB_BEGIN/PIO_TF_AUTO_TEST_SUB_END` tags.
+
+ Fortran code in the stub source file that don't have `PIO_TF_AUTO_TEST_SUB_BEGIN/PIO_TF_AUTO_TEST_SUB_END` tags are not modified by the framework. A test function cannot call other test functions but can call other Fortran utility functions and subroutines. It is however recommended that these utility functions return error codes that can be interpreted by a test function using a parallel assert provided by the framework (to signal a failure in these utility functions).
+
+## Runtime environment
+ The framework also adds code to read command line arguments to set,
+
+* PIO Stride : Set via "--pio-tf-stride" command line option
+* Number of PIO I/O tasks : Set via "--pio-tf-num-io-tasks" command line option
+* Number of PIO aggregators : Set via "--pio-tf-num-aggregators" command line option
+* PIO logging level : Set via "--pio-tf-log-level" command line option
+
+ The above options allow us to run the same test with different PIO parameters.
+
+## Testcase utils
+
+ The following utilities are available from the framework,
+
+*  PIO_TF_CHECK_VAL : Check if two variables are equal. This macro allows checking between two real/integer/double values or arrays. It also allows checking if all the values in an array is equal to a scalar value.
+
+        PIO_TF_CHECK_VAL(scalar_var1, scalar_var2)
+        PIO_TF_CHECK_VAL(array1, array2)
+        PIO_TF_CHECK_VAL(array1, scalar_var1)
+
+* `PIO_TF_CHECK_ERR` : Macro used to check the return code from a PIO function. The format for the macro is shown below,
+
+        PIO_TF_CHECK_ERR(pio_return_code, ``)
+
+* `PIO_TF_PASSERT` : A parallel assert macro. The format is shown below,
+
+        PIO_TF_PASSERT(``, ``)
+
+* `PIO_TF_ASSERT` : An assert (not parallel, local to each process) macro. The format is shown below,
+
+        PIO_TF_ASSERT(``, ``)
+
+* `PIO_TF_LOG` : Allows logging info into stdout
+
+        PIO_TF_LOG(*,*)(``)
+
+* `PIO_TF_ERROR` : Used to flag fatal errors
+
+        PIO_TF_ERROR(``)
+
+* `pio_tf_world_sz_, pio_tf_world_rank_` : Variables that provide size and rank of each process.
+
+## Templated testcases
+
+ Testcases can be templated for the different types supported by PIO. The framework provides a mapped collection of recognized fortran types and the corresponding PIO types. Templated testcases precede with a `PIO_TF_TEMPLATE<>` tag as show below,
+
+    PIO_TF_TEMPLATE
+    PIO_TF_AUTO_TEST_SUB_BEGIN templated_hellow
+
+     PIO_TF_FC_DATA_TYPE, dimension(10) :: buf
+     ...
+
+     call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+     ... 
+
+    PIO_TF_AUTO_tEST_SUB_END templated_hellow
+
+ For the code shown above the framework generates multiple Fortran subroutines by replacing `PIO_TF_FC_DATA_TYPE` and `PIO_TF_DATA_TYPE` with matching Fortran and PIO types respectively.
+
+# Building a testcase
+## Generating the testcase
+
+ Testcases are generated by the '/tests/general/util/pio_tf_f90gen.pl' utility. It parses the stub source code provided as a command line argument and generates Fortran test cases (with all the boiler plate code necessary). The utility can be run as shown below,
+
+
+    ./util/pio_tf_f90gen.pl --annotate-source --out=hellow.F90 hellow.F90.in
+
+
+ In the above command hellow.F90.in is the stub test case created by the developer, and the pio_tf_f90gen.pl utility creates hellow.F90, the test case. The newly generated testcase (hellow.F90) is a Fortran program that depends only on '/tests/general/util/pio_tutil.F90' to build and contains all the code required to run the tests in it.
+
+## Building the generated testcase
+ The test case generated is a Fortran program that depends only on '/tests/general/util/pio_tutil.F90" to build. The test case can be built using any MPI compiler wrapper (mpif90).
+
+ The previous two steps are incorporated into CMakeLists.txt to add a new test to the testsuite.
+
+# Running the testcase
+
+## Running the test suite on your laptop
+ The test suite can be run by invoking the "ctest" command from the PIO build directory
+
+## Running the test suite on Supercomputers/Clusters
+
+ TODO
+
+## Running a single test on your laptop
+ A single test can be run by passing the test executable to an MPI job launcher (mpiexec), similar to running any MPI program.
+
+    mpiexec -n 4 ./hellow
+
+# Adding a test into the test suite
+ The build and running steps need to be incorporated into CMakeLists.txt to add a new testcase into the test suite. Please modify `/tests/general/CMakeLists.txt` as suggested below to add a new test to the test suite,
+
+* Add the testcase stub source into `GENERATED_SRCS`
+* Add a CMake build step : To generate the test executable
+
+        add_executable (pio_init_finalize EXCLUDE_FROM_ALL 
+            pio_init_finalize.F90
+            ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+        target_link_libraries (pio_init_finalize piof)
+        add_dependencies (tests pio_init_finalize)
+
+* Use CMake user defined macro add_mpi_test (or add_test() for mpi-serial) to run the test
+
+        add_mpi_test(init_finialize_1_proc
+                EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+                NUMPROCS 1
+                TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+# Debugging failed tests
+ 
+* Run the test suite using ctest with the "--verbose" option. This outputs a lot of information from the testing framework (to stdout), including the various tests run and the reason for failure. All information in the output from the PIO testing framework is prepended with the `PIO_TF: ` tag. To run a single failing test use the "-R" option available with ctest (e.g. To only run "pio_iosystem_tests3", ctest --verbose -R pio_iosystem_tests3).
+* Run the failing test manually (using the MPI job launcher) and debug it. Also consider running the test with PIO log level > 0 (mpiexec -n 4 ./tests/general/pio_iosystem_tests3 --pio-tf-log-level=6)
diff --git a/externals/pio2/tests/general/ncdf_fail.F90.in b/src/externals/pio2/tests/general/ncdf_fail.F90.in
similarity index 98%
rename from externals/pio2/tests/general/ncdf_fail.F90.in
rename to src/externals/pio2/tests/general/ncdf_fail.F90.in
index f864dfada13..dfc09148a49 100644
--- a/externals/pio2/tests/general/ncdf_fail.F90.in
+++ b/src/externals/pio2/tests/general/ncdf_fail.F90.in
@@ -1,6 +1,6 @@
 MODULE ncdf_fail_tgv
   use pio_tutil
-  implicit none	
+  implicit none
 
   ! tgv = test global vars
   character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_fname = "pio_ncdf_test_file.nc"
@@ -13,12 +13,12 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_clob_then_no_clob
   type(file_desc_t) :: pio_file
   character(len=PIO_TF_MAX_STR_LEN), parameter :: clob_fname = "pio_clob_test_file.nc"
   integer :: ret
-  
+
   ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, clob_fname, PIO_CLOBBER)
   PIO_TF_CHECK_ERR(ret, "Failed to create:" // trim(clob_fname))
 
   call PIO_closefile(pio_file)
-  
+
   ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, clob_fname, PIO_NOCLOBBER)
   PIO_TF_PASSERT(ret /= PIO_NOERR, "Create file with clobber then no clobber did not fail as expected")
 
@@ -32,7 +32,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_redef_with_no_write
   Implicit none
   type(file_desc_t) :: pio_file
   integer :: ret
-  
+
   ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_nowrite)
   PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
 
@@ -50,7 +50,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_redef_twice
   Implicit none
   type(file_desc_t) :: pio_file
   integer :: ret
-  
+
   ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_write)
   PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
 
@@ -79,7 +79,7 @@ PIO_TF_TEST_DRIVER_BEGIN
   integer :: num_iotypes
 
   num_iotypes = 0
-  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) 
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
   do i=1,num_iotypes
     tgv_iotype = iotypes(i)
     ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname)
@@ -92,7 +92,7 @@ PIO_TF_TEST_DRIVER_BEGIN
     PIO_TF_AUTO_TESTS_RUN(trim(iotype_descs(i)))
 
     call PIO_deletefile(pio_tf_iosystem_, tgv_fname)
-  end do 
+  end do
   if(allocated(iotypes)) then
     deallocate(iotypes)
     deallocate(iotype_descs)
diff --git a/src/externals/pio2/tests/general/ncdf_get_put.F90.in b/src/externals/pio2/tests/general/ncdf_get_put.F90.in
new file mode 100644
index 00000000000..75ccc75f3e9
--- /dev/null
+++ b/src/externals/pio2/tests/general/ncdf_get_put.F90.in
@@ -0,0 +1,269 @@
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_1datt
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: val
+  CHARACTER(len=DIM_LEN) :: cval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  val = pio_tf_world_sz_
+  cval = "DUMMY_STRING"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_cval', PIO_char, (/pio_dim/), pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_var, 'dummy_att_put_val', val);
+    PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_cvar, 'dummy_att_put_cval', cval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put char attribute:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_1datt
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1datt
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval
+  PIO_TF_FC_DATA_TYPE :: init_val
+  CHARACTER(len=DIM_LEN) :: pcval, gcval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  init_val = pio_tf_world_sz_
+
+  pval = init_val
+  pcval = "DUMMY_STRING"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_cval', PIO_char, (/pio_dim/), pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_var, 'dummy_att_val', pval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_cvar, 'dummy_att_cval', pcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put char attribute:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ret = PIO_get_att(pio_file, pio_var, 'dummy_att_val', gval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get attribute:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, init_val), "Got wrong value")
+
+    ! FIXME: Check the values are correct
+    ret = PIO_get_att(pio_file, pio_cvar, 'dummy_att_cval', gcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get attribute:" // trim(filename))
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1datt
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval
+  CHARACTER(len=DIM_LEN) :: pcval, gcval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  pval = pio_tf_world_sz_
+  pcval = "DUMMY_STRING"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_cval', PIO_char, (/pio_dim/), pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_var, pval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put var:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_cvar, pcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put char var:" // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    ret = PIO_get_var(pio_file, pio_var, gval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, pval), "Got wrong value")
+
+    ret = PIO_get_var(pio_file, pio_cvar, gcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get char var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gcval, pcval), "Got wrong value")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_4parts
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 16
+  integer, parameter :: PART_LEN = DIM_LEN / 4
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  integer, dimension(1) :: start, count = PART_LEN
+  
+  do i=1,DIM_LEN
+    pval(i) = i
+  end do
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    start = 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(1 : PART_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (1st part):" // trim(filename))
+
+    start = PART_LEN + 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(PART_LEN + 1 : PART_LEN * 2))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (2nd part):" // trim(filename))
+
+    start = PART_LEN * 2 + 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(PART_LEN * 2 + 1 : PART_LEN * 3))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (3rd part):" // trim(filename))
+
+    start = PART_LEN * 3 + 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(PART_LEN * 3 + 1 : DIM_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (4th part):" // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    start = PART_LEN * 3 + 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(PART_LEN * 3 + 1 : DIM_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (4th part):" // trim(filename))
+
+    start = PART_LEN * 2 + 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(PART_LEN * 2 + 1 : PART_LEN * 3))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (3rd part):" // trim(filename))
+
+    start = PART_LEN + 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(PART_LEN + 1 : PART_LEN * 2))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (2nd part):" // trim(filename))
+
+    start = 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(1 : PART_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (1st part):" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, pval), "Got wrong value")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_4parts
+
diff --git a/src/externals/pio2/tests/general/ncdf_inq.F90.in b/src/externals/pio2/tests/general/ncdf_inq.F90.in
new file mode 100644
index 00000000000..6ac3b1fd3ce
--- /dev/null
+++ b/src/externals/pio2/tests/general/ncdf_inq.F90.in
@@ -0,0 +1,192 @@
+MODULE ncdf_inq_tests_tgv
+  use pio_tutil
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_fname = "pio_ncdf_inq_test_file.nc"
+  integer :: tgv_iotype
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_var_name = "dummy_var"
+  integer, parameter :: tgv_var_ndims = 1
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_dim_name = "dummy_dim"
+  integer, parameter :: TGV_DIM_LEN = 100
+  integer, parameter :: tgv_var_natts = 2
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_iatt_name = "dummy_iatt"
+  integer, parameter :: tgv_iatt_val = 3
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_catt_name = "dummy_catt"
+  integer, parameter :: TGV_ATT_LEN = 100
+  character(len=TGV_ATT_LEN) :: tgv_catt_val = "DUMMY_STR"
+END MODULE ncdf_inq_tests_tgv
+
+SUBROUTINE test_setup(ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  integer, intent(out) :: ret
+
+  type(file_desc_t) :: pio_file
+  type(var_desc_t)  :: pio_var
+  integer :: pio_dim
+  real, dimension(TGV_DIM_LEN) :: val
+  integer, dimension(TGV_ATT_LEN) :: iatt
+  CHARACTER(len=TGV_ATT_LEN) :: catt
+
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
+
+  ! Since file is just created no need to enter redef
+  ret = PIO_def_dim(pio_file, tgv_dim_name, TGV_DIM_LEN, pio_dim)
+  PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(tgv_fname))
+
+  ret = PIO_def_var(pio_file, tgv_var_name, pio_real, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(tgv_fname))
+
+  iatt = tgv_iatt_val
+  ret = PIO_put_att(pio_file, pio_var, tgv_iatt_name, iatt);
+  PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(tgv_fname))
+
+  catt = tgv_catt_val
+  ret = PIO_put_att(pio_file, pio_var, tgv_catt_name, catt);
+  PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(tgv_fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(tgv_fname))
+
+  val = pio_tf_world_rank_
+  ret = PIO_put_var(pio_file, pio_var, val)
+  PIO_TF_CHECK_ERR(ret, "Failed to put var: " // trim(tgv_fname))
+
+  call PIO_closefile(pio_file)
+END SUBROUTINE test_setup
+
+SUBROUTINE test_teardown(ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  integer, intent(out) :: ret
+
+  ret = PIO_NOERR
+  call PIO_deletefile(pio_tf_iosystem_, tgv_fname)  
+END SUBROUTINE test_teardown
+
+SUBROUTINE test_inq_var(pio_file, ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  type(file_desc_t), intent(in) :: pio_file
+  integer, intent(inout) :: ret
+
+  type(var_desc_t)  :: pio_var
+  integer :: var_id, var_type, var_ndims, var_natts
+  integer, dimension(:), allocatable :: var_dims
+  character(len=pio_max_name) :: var_name
+
+  ret = pio_inq_varid(pio_file, tgv_var_name, var_id)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire varid :"//trim(tgv_fname))
+
+  ret = pio_inq_varname(pio_file, var_id, var_name)
+  PIO_TF_PASSERT(var_name .eq. tgv_var_name, "Variable name is not the expected value")
+
+  ret = pio_inq_vartype(pio_file, var_id, var_type)
+  PIO_TF_PASSERT(var_type == PIO_real, "Variable tye is not the expected type")
+
+  ret = pio_inq_varndims(pio_file, var_id, var_ndims)
+  PIO_TF_PASSERT(var_ndims == tgv_var_ndims, "Num of dims for variable is not expected")
+
+  allocate(var_dims(var_ndims))
+  ret = pio_inq_vardimid(pio_file, var_id, var_dims)
+  PIO_TF_CHECK_ERR(ret, "Failed to get dim ids:"//trim(tgv_fname))
+  deallocate(var_dims)
+
+  ret = pio_inq_varnatts(pio_file, var_id, var_natts)
+  PIO_TF_PASSERT(var_natts == tgv_var_natts, "Num of atts for variable is not expected")
+
+END SUBROUTINE test_inq_var
+
+SUBROUTINE test_inq_dim(pio_file, ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  type(file_desc_t), intent(in) :: pio_file
+  integer, intent(inout) :: ret
+
+  integer :: dim_id
+  character(len=pio_max_name) :: dim_name
+  integer(kind=pio_offset_kind) :: dim_len
+
+  ret = pio_inq_dimid(pio_file, tgv_dim_name, dim_id)
+  PIO_TF_CHECK_ERR(ret, "Failed to inq dimid :"//trim(tgv_fname))
+
+  ret = pio_inq_dimname(pio_file, dim_id, dim_name)
+  PIO_TF_PASSERT(dim_name .eq. tgv_dim_name, "Dim name is not the expected value") 
+
+  ret = pio_inq_dimlen(pio_file, dim_id, dim_len)
+  PIO_TF_PASSERT(dim_len == TGV_DIM_LEN, "Dim length is not the expected value")
+
+END SUBROUTINE test_inq_dim
+
+SUBROUTINE test_inq_att(pio_file, ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  type(file_desc_t), intent(inout) :: pio_file
+  integer, intent(inout) :: ret
+
+  integer :: var_id
+  integer(kind=pio_offset_kind) :: att_len
+
+  ret = pio_inq_varid(pio_file, tgv_var_name, var_id)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire varid :"//trim(tgv_fname))
+
+  ret = pio_inq_attlen(pio_file, var_id, tgv_iatt_name, att_len)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire att len :"//trim(tgv_fname))
+  !PIO_TF_PASSERT(att_len == tgv_iatt_len, "Att length is not expected value")
+
+  ret = pio_inq_attlen(pio_file, var_id, tgv_catt_name, att_len)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire att len :"//trim(tgv_fname))
+  PIO_TF_PASSERT(att_len == len(trim(tgv_catt_val)), "Attribute length is not expected value")
+
+END SUBROUTINE test_inq_att
+
+PIO_TF_AUTO_TEST_SUB_BEGIN test_inq
+  use ncdf_inq_tests_tgv
+  Implicit none
+  type(file_desc_t) :: pio_file
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    tgv_iotype = iotypes(i)
+
+    call test_setup(ret)
+    PIO_TF_CHECK_ERR(ret, "Test setup failed")
+
+    ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_nowrite)
+    PIO_TF_CHECK_ERR(ret, "Failed to open: "// trim(tgv_fname))
+
+    call test_inq_var(pio_file, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to inq var:" // trim(tgv_fname))
+
+    call test_inq_dim(pio_file, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to inq dim:" // trim(tgv_fname))
+
+    call test_inq_att(pio_file, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to inq att:" // trim(tgv_fname))
+
+    call PIO_closefile(pio_file)
+
+    call test_teardown(ret)
+    PIO_TF_CHECK_ERR(ret, "Test teardown failed")
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_inq
diff --git a/externals/pio2/tests/general/ncdf_simple_tests.F90.in b/src/externals/pio2/tests/general/ncdf_simple_tests.F90.in
similarity index 100%
rename from externals/pio2/tests/general/ncdf_simple_tests.F90.in
rename to src/externals/pio2/tests/general/ncdf_simple_tests.F90.in
diff --git a/externals/pio2/tests/general/pio_decomp_fillval.F90.in b/src/externals/pio2/tests/general/pio_decomp_fillval.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_decomp_fillval.F90.in
rename to src/externals/pio2/tests/general/pio_decomp_fillval.F90.in
diff --git a/externals/pio2/tests/general/pio_decomp_fillval2.F90 b/src/externals/pio2/tests/general/pio_decomp_fillval2.F90
similarity index 100%
rename from externals/pio2/tests/general/pio_decomp_fillval2.F90
rename to src/externals/pio2/tests/general/pio_decomp_fillval2.F90
diff --git a/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in
new file mode 100644
index 00000000000..e92b483c3c0
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in
@@ -0,0 +1,227 @@
+! Get a 3D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_HGT_SZ blocks of 
+! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements
+! # All odd procs have  VEC_HGT_SZ blocks of
+! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1) |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1  |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! This for example can be used to force rearrangement when reading
+! or writing data.
+SUBROUTINE get_3d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 2
+  integer, parameter :: VEC_COL_SZ = 2
+  integer, parameter :: VEC_HGT_SZ = 2
+  integer, parameter :: NDIMS = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  dims(3) = VEC_HGT_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+  ! Height
+  start(3) = 1
+  count(3) = VEC_HGT_SZ
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_4d_col_decomp
+  implicit none
+  integer, parameter :: NDIMS = 4
+  integer, parameter :: NFRAMES = 3
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:,:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS-1) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts
+  integer(kind=pio_offset_kind) :: f
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(wbuf(nrows, ncols, nhgts, NFRAMES))
+  allocate(compdof(nrows * ncols * nhgts))
+  do f=1,NFRAMES
+    do k=1,nhgts
+      do j=1,ncols
+        do i=1,nrows
+          wbuf(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                        (start(2) - 1 + j - 1) * dims(1) + i
+          wbuf(i,j,k,f) = wbuf(i,j,k,f) + (f - 1) * (dims(1) * dims(2) * dims(3))
+          tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+          compdof(tmp_idx) = wbuf(i,j,k,1)
+        end do
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(rbuf(nrows, ncols, nhgts, NFRAMES))
+  allocate(compdof(nrows * ncols * nhgts))
+  allocate(exp_val(nrows, ncols, nhgts, NFRAMES))
+
+  do f=1,NFRAMES
+    do k=1,nhgts
+      do j=1,ncols
+        do i=1,nrows
+          tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+          compdof(tmp_idx) =  (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                              (start(2) - 1 + j - 1) * dims(1) + i
+          exp_val(i,j,k,f) = compdof(tmp_idx) + (f - 1) * (dims(1) * dims(2) * dims(3))
+        end do
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(4))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    do f=1,NFRAMES
+      call PIO_setframe(pio_file, pio_var, f)
+      ! Write the current frame
+      call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf(:,:,:,f), ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+    end do
+
+    call PIO_syncfile(pio_file)
+
+    do f=1,NFRAMES
+      call PIO_setframe(pio_file, pio_var, f)
+      call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf(:,:,:,f), ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+    end do
+
+    do f=1,NFRAMES
+      PIO_TF_CHECK_VAL((rbuf(:,:,:,f), exp_val(:,:,:,f)), "Got wrong val, frame=", f)
+    end do
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_4d_col_decomp
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests.F90.in
new file mode 100644
index 00000000000..40476f4740a
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests.F90.in
@@ -0,0 +1,305 @@
+PIO_TF_AUTO_TEST_SUB_BEGIN init_decomp_1d_get_loc_sz
+  implicit none
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, dimension(:), allocatable :: data_types
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: data_type_descs
+  type(io_desc_t) :: iodesc
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  integer, dimension(1) :: dims
+  integer :: i, ntypes, lsz
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+
+  call PIO_TF_Get_data_types(data_types, data_type_descs, ntypes)
+  do i=1,ntypes
+    PIO_TF_LOG(0,*) "Testing type : ", data_type_descs(i)
+    call PIO_initdecomp(pio_tf_iosystem_, data_types(i), dims, compdof, iodesc)
+    lsz = PIO_get_local_array_size(iodesc)
+    PIO_TF_PASSERT(lsz == VEC_LOCAL_SZ, "Checking the local array size")
+    call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+  end do
+
+  if(allocated(data_types)) then
+    deallocate(data_types)
+    deallocate(data_type_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END init_decomp_1d_get_loc_sz
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_darray
+  implicit none
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: iodesc
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: buf
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+  buf = pio_tf_world_rank_;
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    ! FIXME: Verify the written output
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+PIO_TF_AUTO_TEST_SUB_END nc_write_1d_darray
+
+! Write 1d array, although diff procs have different
+! number of elements to write locally they all use
+! the same buffer size (compdof size is different for
+! each rank but buf size is the same)
+! Odd procs write out VEC_LOCAL_SZ_ODD elements &
+! even procs write out VEC_LOCAL_SZ_EVEN elements, but
+! all procs use buf[MAX_VEC_SZ]
+! eg:
+! Elements in buffer on each proc with MAX_VEC_SZ = 2,
+! [0 1] [2 X] [3 4] [5 X] ...
+! The 'X'es in the buffer are not written out
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_const_buf_sz
+  implicit none
+  integer, parameter :: MAX_VEC_SZ = 2
+  integer, parameter :: VEC_LOCAL_SZ_ODD = MAX_VEC_SZ - 1
+  integer, parameter :: VEC_LOCAL_SZ_EVEN = MAX_VEC_SZ
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: iodesc
+  integer, dimension(:), allocatable :: compdof, compdof_rel_disps
+  integer :: compdof_rel_start
+  integer :: cdof_sz = VEC_LOCAL_SZ_ODD
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_VEC_SZ) :: wbuf, rbuf
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  logical :: is_even = .false.
+  integer :: nodd_procs, nodd_procs_bfr, neven_procs, neven_procs_bfr
+
+  nodd_procs = pio_tf_world_sz_ / 2
+  ! Number of odd procs before this rank
+  nodd_procs_bfr = pio_tf_world_rank_ / 2
+  neven_procs = pio_tf_world_sz_ - nodd_procs
+  ! Number of even procs before this rank
+  neven_procs_bfr = pio_tf_world_rank_ - nodd_procs_bfr
+
+  ! Odd procs write out VEC_LOCAL_SZ_ODD elements &
+  ! even procs write out VEC_LOCAL_SZ_EVEN elements
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    cdof_sz = VEC_LOCAL_SZ_EVEN
+  end if
+  allocate(compdof(cdof_sz))
+  allocate(compdof_rel_disps(cdof_sz))
+  wbuf = 0
+  rbuf = 0
+  do i=1,cdof_sz
+    compdof_rel_disps(i) = i
+    wbuf(i) = i
+  end do
+  ! Find out where compdof starts for this rank
+  compdof_rel_start = nodd_procs_bfr * VEC_LOCAL_SZ_ODD +&
+                      neven_procs_bfr * VEC_LOCAL_SZ_EVEN
+  dims(1) = nodd_procs * VEC_LOCAL_SZ_ODD + neven_procs * VEC_LOCAL_SZ_EVEN
+  compdof = compdof_rel_start + compdof_rel_disps
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+  deallocate(compdof)
+  deallocate(compdof_rel_disps)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_const_buf_sz
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_reuse_decomp
+  implicit none
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  type(var_desc_t)  :: pio_var1_file1, pio_var2_file1, pio_var1_file2
+  type(file_desc_t) :: pio_file1, pio_file2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename1, filename2
+  type(io_desc_t) :: iodesc
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: buf, rbuf
+  integer, dimension(1) :: dims
+  integer :: pio_dim_file1, pio_dim_file2
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+  buf = pio_tf_world_rank_;
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename1 = "test_pio_decomp_simple_tests.testfile1"
+  filename2 = "test_pio_decomp_simple_tests.testfile2"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file1, iotypes(i),&
+                            filename1, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1))
+
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file2, iotypes(i),&
+                            filename2, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1))
+
+    ierr = PIO_def_dim(pio_file1, 'PIO_TF_test_dim', dims(1), pio_dim_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename1))
+
+    ierr = PIO_def_dim(pio_file2, 'PIO_TF_test_dim', dims(1), pio_dim_file2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename2))
+
+    ierr = PIO_def_var(pio_file1, 'PIO_TF_test_var1', PIO_TF_DATA_TYPE,&
+                          (/pio_dim_file1/), pio_var1_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define first var : " // trim(filename1))
+
+    ierr = PIO_def_var(pio_file1, 'PIO_TF_test_var2', PIO_TF_DATA_TYPE,&
+                          (/pio_dim_file1/), pio_var2_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define second var : " // trim(filename1))
+
+    ierr = PIO_def_var(pio_file2, 'PIO_TF_test_var1', PIO_TF_DATA_TYPE,&
+                          (/pio_dim_file2/), pio_var1_file2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define first var : " // trim(filename2))
+
+    ierr = PIO_enddef(pio_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename1))
+
+    ierr = PIO_enddef(pio_file2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename2))
+
+    ! Write the variables - file1
+    call PIO_write_darray(pio_file1, pio_var1_file1, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write first darray : " // trim(filename1))
+
+    call PIO_write_darray(pio_file1, pio_var2_file1, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write second darray : " // trim(filename1))
+
+    ! Write the variables - file2
+    call PIO_write_darray(pio_file2, pio_var1_file2, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write first darray : " // trim(filename2))
+
+    call PIO_syncfile(pio_file1)
+    rbuf = 0
+    call PIO_read_darray(pio_file1, pio_var1_file1, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read first darray : " // trim(filename1))
+
+    PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+
+    rbuf = 0
+    call PIO_read_darray(pio_file1, pio_var2_file1, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read second darray : " // trim(filename1))
+
+    PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+
+    call PIO_syncfile(pio_file2)
+    rbuf = 0
+    call PIO_read_darray(pio_file2, pio_var1_file2, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read first darray : " // trim(filename2))
+
+    PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+
+    call PIO_closefile(pio_file1)
+    call PIO_closefile(pio_file2)
+    call PIO_deletefile(pio_tf_iosystem_, filename1);
+    call PIO_deletefile(pio_tf_iosystem_, filename2);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+PIO_TF_AUTO_TEST_SUB_END nc_write_1d_reuse_decomp
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in
new file mode 100644
index 00000000000..48d4f95a101
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in
@@ -0,0 +1,306 @@
+! Get a block cyclic decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ elements
+! # All odd procs have VEC_LOCAL_SZ + 1 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4,5] [6,7]
+! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
+! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [3,4,5] [1,2] [6,7]
+! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
+! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
+SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_LOCAL_SZ + 1
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
+      else
+        count(1) = VEC_LOCAL_SZ
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_LOCAL_SZ
+      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_LOCAL_SZ
+    else
+      count(1) = VEC_LOCAL_SZ + 1
+    end if
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Get a 1d block decomposition with holes
+! If has_hole is TRUE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ * 2 elements
+! # (the even procs take all elems from the odd procs)
+! # All odd procs have 0 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2,3,4] [] [5,6]
+! e.g. 2)    [1,2,3,4] [] [5,6,7,8] []
+! If has_hole is FALSE, the data is evenly divided among all procs
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4] [5,6]
+! e.g. 2)    [1,2] [3,4] [5,6] [7,8]
+SUBROUTINE get_1d_bc_with_holes_info(rank, sz, dims, start, count, has_hole)
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: has_hole
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_LOCAL_SZ * sz
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(has_hole) then
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        count(1) = VEC_LOCAL_SZ * 2
+      else
+        count(1) = VEC_LOCAL_SZ
+      end if
+      start(1) = ieven * VEC_LOCAL_SZ * 2 + 1
+    else
+      count(1) = 0
+      start(1) = 0
+    end if
+  else
+    count(1) = VEC_LOCAL_SZ
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ) + 1
+  end if
+
+END SUBROUTINE
+
+! Test block cyclic interface
+! Write with one decomp and read with another
+! Test all combs 
+! - no rearrage read + no rearrange write
+! - rearrage read + no rearrange write
+! - no rearrage read + rearrange write
+! - rearrage read + rearrange write
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_rd_1d_bc
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  logical, dimension(2) :: enable_rd_rearr = (/.false., .true./)
+  integer :: rd_rearr_opt_idx
+  logical, dimension(2) :: enable_wr_rearr = (/.false., .true./)
+  integer :: wr_rearr_opt_idx
+
+  do rd_rearr_opt_idx=1,size(enable_rd_rearr)
+    do wr_rearr_opt_idx=1,size(enable_wr_rearr)
+      PIO_TF_LOG(0, *) "Testing Rd rearr =", enable_rd_rearr(rd_rearr_opt_idx), ",Write rearr=", enable_wr_rearr(wr_rearr_opt_idx)
+      ! Set the decomposition for writing data - forcing rearrangement
+      call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
+             start, count, enable_wr_rearr(wr_rearr_opt_idx))
+      allocate(wbuf(count(1)))
+      allocate(compdof(count(1)))
+      do i=1,count(1)
+        wbuf(i) = start(1) + i - 1
+        compdof(i) = start(1) + i - 1
+      end do
+
+      call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+      deallocate(compdof)
+
+      ! Set the decomposition for reading data - different from the write decomp
+      call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
+             start, count, enable_rd_rearr(rd_rearr_opt_idx))
+      allocate(rbuf(count(1)))
+      allocate(compdof(count(1)))
+      allocate(exp_val(count(1)))
+      do i=1,count(1)
+        compdof(i) = start(1) + i -1
+        ! Expected value, after reading, is the same as the compdof
+        exp_val(i) = compdof(i)
+      end do
+
+      call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+      deallocate(compdof)
+
+      num_iotypes = 0
+      call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+      filename = "test_pio_decomp_simple_tests.testfile"
+      do i=1,num_iotypes
+        PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+        ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+        PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+        ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+        ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+        ierr = PIO_enddef(pio_file)
+        PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+        ! Write the variable out
+        call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+        call PIO_syncfile(pio_file)
+
+        rbuf = 0
+        call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+        PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+        call PIO_closefile(pio_file)
+        
+        call PIO_deletefile(pio_tf_iosystem_, filename);
+      end do
+
+      if(allocated(iotypes)) then
+        deallocate(iotypes)
+        deallocate(iotype_descs)
+      end if
+
+      call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+      call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+      deallocate(exp_val)
+      deallocate(rbuf)
+      deallocate(wbuf)
+    end do !wr_rearr_opt_idx=1,size(enable_wr_rearr)
+  end do !rd_rearr_opt_idx=1,size(enable_rd_rearr)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_rd_1d_bc
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - has holes
+  call get_1d_bc_with_holes_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  allocate(wbuf(count(1)))
+  allocate(rbuf(count(1)))
+  allocate(compdof(count(1)))
+  allocate(exp_val(count(1)))
+  do i=1,count(1)
+    wbuf(i) = start(1) + i - 1
+    compdof(i) = wbuf(i)
+    exp_val(i) = wbuf(i)
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, wr_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_bc_with_holes
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in
new file mode 100644
index 00000000000..27dafcdfb0b
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in
@@ -0,0 +1,367 @@
+! Get a 2D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements
+! # All odd procs have  VEC_COL_SZ rows of VEC_ROW_SZ + 1 elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2,
+! e.g. 1) |(1,1) (1,2)|  |(1,3) (1,4) (1,5)|  |(1,6) (1,7)|
+!         |(2,1) (2,2)|, |(2,3) (2,4) (2,5)|, |(2,6) (2,7)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2
+! e.g. 1) |(1,3) (1,4) (1,5)|  |(1,1) (1,2)|   |(1,6) (1,7)|
+!         |(2,3) (2,4) (2,5)|, |(2,1) (2,2)|,  |(2,6) (2,7)|
+SUBROUTINE get_2d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 7
+  integer, parameter :: VEC_COL_SZ = 7
+  integer, parameter :: NDIMS = 2
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Get a 2D row decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements
+! # All odd procs have  VEC_COL_SZ+1 rows of VEC_ROW_SZ elements
+! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2,
+! e.g. 1) |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|,
+!         |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)|
+!         |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|,
+!         |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements (rows) with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2
+! e.g. 1) |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)|
+!         |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|,
+!         |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|,
+!         |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)|
+SUBROUTINE get_2d_row_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_COL_SZ = 7
+  integer, parameter :: VEC_ROW_SZ = 7
+  integer, parameter :: NDIMS = 2
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_COL_SZ + num_odd_procs * (VEC_COL_SZ + 1)
+  dims(2) = VEC_ROW_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_COL_SZ + 1
+        start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + (VEC_COL_SZ) + 1
+      else
+        count(1) = VEC_COL_SZ
+        start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_COL_SZ
+      start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) - (VEC_COL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_COL_SZ
+    else
+      count(1) = VEC_COL_SZ + 1
+    end if
+    start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1
+  end if
+
+  ! Columns
+  start(2) = 1
+  count(2) = VEC_ROW_SZ
+
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_col_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, tmp_idx, ierr, lsz, nrows, ncols
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  
+  allocate(wbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  do j=1,ncols
+    do i=1,nrows
+      wbuf(i,j) = (start(2) - 1 + j - 1) * nrows + i
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = wbuf(i,j)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  
+  allocate(rbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  allocate(exp_val(nrows, ncols))
+  do j=1,ncols
+    do i=1,nrows
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = (start(2) - 1 + j - 1) * nrows + i
+      ! Expected value, after reading, is the same as the compdof
+      exp_val(i,j) = compdof(tmp_idx)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_col_decomp
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_row_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, tmp_idx, ierr, lsz, nrows, ncols
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+
+  allocate(wbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  do j=1,ncols
+    do i=1,nrows
+      wbuf(i,j) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = wbuf(i,j)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  
+  allocate(rbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  allocate(exp_val(nrows, ncols))
+  do j=1,ncols
+    do i=1,nrows
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1
+      ! Expected value, after reading, is the same as the compdof
+      exp_val(i,j) = compdof(tmp_idx)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_row_decomp
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in
new file mode 100644
index 00000000000..22cdacfe98a
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in
@@ -0,0 +1,209 @@
+! Get a 3D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_HGT_SZ blocks of 
+! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements
+! # All odd procs have  VEC_HGT_SZ blocks of
+! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1) |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1  |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! This for example can be used to force rearrangement when reading
+! or writing data.
+SUBROUTINE get_3d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 2
+  integer, parameter :: VEC_COL_SZ = 2
+  integer, parameter :: VEC_HGT_SZ = 2
+  integer, parameter :: NDIMS = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  dims(3) = VEC_HGT_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+  ! Height
+  start(3) = 1
+  count(3) = VEC_HGT_SZ
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 3
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(wbuf(nrows, ncols, nhgts))
+  allocate(compdof(nrows * ncols * nhgts))
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        wbuf(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                      (start(2) - 1 + j - 1) * dims(1) + i
+        tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+        compdof(tmp_idx) = wbuf(i,j,k)
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(rbuf(nrows, ncols, nhgts))
+  allocate(compdof(nrows * ncols * nhgts))
+  allocate(exp_val(nrows, ncols, nhgts))
+
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+        compdof(tmp_idx) =  (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                            (start(2) - 1 + j - 1) * dims(1) + i
+        exp_val(i,j,k) = compdof(tmp_idx)
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_3d_col_decomp
diff --git a/externals/pio2/tests/general/pio_fail.F90.in b/src/externals/pio2/tests/general/pio_fail.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_fail.F90.in
rename to src/externals/pio2/tests/general/pio_fail.F90.in
diff --git a/externals/pio2/tests/general/pio_file_fail.F90.in b/src/externals/pio2/tests/general/pio_file_fail.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_file_fail.F90.in
rename to src/externals/pio2/tests/general/pio_file_fail.F90.in
diff --git a/externals/pio2/tests/general/pio_file_simple_tests.F90.in b/src/externals/pio2/tests/general/pio_file_simple_tests.F90.in
similarity index 97%
rename from externals/pio2/tests/general/pio_file_simple_tests.F90.in
rename to src/externals/pio2/tests/general/pio_file_simple_tests.F90.in
index 05f90fd7f9a..af5604b1798 100644
--- a/externals/pio2/tests/general/pio_file_simple_tests.F90.in
+++ b/src/externals/pio2/tests/general/pio_file_simple_tests.F90.in
@@ -53,7 +53,7 @@ PIO_TF_TEST_DRIVER_BEGIN
   integer :: num_iotypes
 
   num_iotypes = 0
-  call PIO_TF_Get_iotypes(iotypes, iotype_descs, num_iotypes) 
+  call PIO_TF_Get_iotypes(iotypes, iotype_descs, num_iotypes)
   dummy_file = "test_pio_file_simple_tests.testfile"
   do i=1,num_iotypes
     PIO_TF_TEST_RUN(create_file_no_opts(iotypes(i), dummy_file), trim(iotype_descs(i)))
@@ -66,4 +66,3 @@ PIO_TF_TEST_DRIVER_BEGIN
   end if
 
 PIO_TF_TEST_DRIVER_END
-
diff --git a/externals/pio2/tests/general/pio_init_finalize.F90.in b/src/externals/pio2/tests/general/pio_init_finalize.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_init_finalize.F90.in
rename to src/externals/pio2/tests/general/pio_init_finalize.F90.in
diff --git a/src/externals/pio2/tests/general/pio_iosystem_tests.F90.in b/src/externals/pio2/tests/general/pio_iosystem_tests.F90.in
new file mode 100644
index 00000000000..33e59a2c96b
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_iosystem_tests.F90.in
@@ -0,0 +1,309 @@
+! Split comm world into two comms (one with even procs and the other
+! with odd procs
+SUBROUTINE split_world_odd_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = 0
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  call MPI_Comm_rank(new_comm, new_rank, ierr)
+  call MPI_Comm_size(new_comm, new_size, ierr)
+END SUBROUTINE split_world_odd_even
+
+SUBROUTINE split_world_only_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = MPI_UNDEFINED
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  if(new_comm /= MPI_COMM_NULL) then
+    call MPI_Comm_rank(new_comm, new_rank, ierr)
+    call MPI_Comm_size(new_comm, new_size, ierr)
+  end if
+END SUBROUTINE split_world_only_even
+
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Check the contents of file : Check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+    character(len=PIO_TF_MAX_STR_LEN) :: val
+
+    ret = PIO_inq_dimid(pio_file, dimname, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_inq_varid(pio_file, attname, pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_get_att(pio_file, pio_var, attname, val)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to get att " // trim(attname) // " in file :" // trim(fname))
+
+    PRINT *, "val = ", trim(val), ", fname =", trim(fname)
+    PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
+END SUBROUTINE check_file
+
+! Open and check the contents of file : open it and check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
+                      attname, dimname, disable_fclose, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    logical, intent(in) :: disable_fclose
+    integer, intent(inout) :: ret
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    call check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
+
+    if(.not. disable_fclose) then
+      call PIO_closefile(pio_file)
+    end if
+END SUBROUTINE open_and_check_file
+
+! Create a file with one iosystem - with all procs, and open/read with
+! another iosystem - subset (odd/even) of procs
+PIO_TF_AUTO_TEST_SUB_BEGIN two_iosystems_odd_even
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, num_iotypes = 0
+  type(file_desc_t) :: pio_file
+
+  type(iosystem_desc_t) :: odd_even_iosys
+  integer :: odd_even_comm, odd_even_comm_rank, odd_even_comm_size
+  logical :: is_even
+  integer :: ret
+
+  ! Split world to odd and even procs
+  call split_world_odd_even(odd_even_comm, odd_even_comm_rank, odd_even_comm_size, is_even)
+
+  call PIO_init(odd_even_comm_rank, odd_even_comm, odd_even_comm_size, &
+                1, &! Num aggregators
+                1, &! Stride
+                PIO_rearr_subset, odd_even_iosys, base=0)
+  call PIO_seterrorhandling(odd_even_iosys, PIO_BCAST_ERROR)
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create two files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file1 from odd processes and file2 from even processes
+    if(is_even) then
+      fname => fname1
+    else
+      fname => fname2
+    end if
+
+    call open_and_check_file(odd_even_comm, odd_even_iosys, iotypes(i), &
+                    pio_file, fname, attname, dimname, .false., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname)
+  end do
+
+  call PIO_finalize(odd_even_iosys, ret)
+  call MPI_Comm_free(odd_even_comm, ret)
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END two_iosystems_odd_even
+
+! Create a file with one iosystem - with all procs, and open/read with
+! two iosystems - (1) with all procs (2) subset (even) of procs
+PIO_TF_AUTO_TEST_SUB_BEGIN two_iosystems_even_all
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, num_iotypes = 0
+  type(file_desc_t) :: pio_file1, pio_file2
+
+  type(iosystem_desc_t) :: odd_even_iosys
+  integer :: odd_even_comm, odd_even_comm_rank, odd_even_comm_size
+  logical :: is_even
+  logical :: disable_fclose = .true.
+  integer :: ret
+
+  ! Split world to odd and even procs
+  call split_world_only_even(odd_even_comm, odd_even_comm_rank, odd_even_comm_size, is_even)
+
+  if(is_even) then
+    call PIO_init(odd_even_comm_rank, odd_even_comm, odd_even_comm_size, &
+                  1, &! Num aggregators
+                  1, &! Stride
+                  PIO_rearr_subset, odd_even_iosys, base=0)
+    call PIO_seterrorhandling(odd_even_iosys, PIO_BCAST_ERROR)
+  end if
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create two files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file1 from all even processes and file2 from all odd processes
+    ! - PIO called from all processes with the odd_even_iosys
+    if(is_even) then
+      call open_and_check_file(odd_even_comm, odd_even_iosys, iotypes(i), &
+                    pio_file1, fname1, attname, dimname, disable_fclose, ret)
+      PRINT *, "file1,", trim(fname1), "fh: ", pio_file1%fh
+    end if
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname1)
+
+    call open_and_check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                    pio_file2, fname2, attname, dimname, disable_fclose, ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname2)
+    PRINT *, "file2,", trim(fname2), "fh: ", pio_file2%fh
+
+    ! Check contents of the files again
+    ! - PIO called from odd and even processes separately with odd_even_iosys
+    if(is_even) then
+      call check_file(odd_even_comm, odd_even_iosys, iotypes(i), pio_file1, &
+                      fname1, attname, dimname, ret)
+      !call PIO_closefile(pio_file1)
+    end if
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname1)
+
+    call check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), pio_file2, &
+                    fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname2)
+
+    if(disable_fclose) then
+      if(is_even) then
+        call PIO_closefile(pio_file1)
+      end if
+      call PIO_closefile(pio_file2)
+    end if
+  end do
+
+  if(is_even) then
+    call PIO_finalize(odd_even_iosys, ret)
+    call MPI_Comm_free(odd_even_comm, ret)
+  end if
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END two_iosystems_even_all
diff --git a/src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in b/src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in
new file mode 100644
index 00000000000..e8e3d8fe431
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in
@@ -0,0 +1,238 @@
+! Split comm world into two comms (one with even procs and the other
+! with odd procs
+SUBROUTINE split_world_odd_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = 0
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  call MPI_Comm_rank(new_comm, new_rank, ierr)
+  call MPI_Comm_size(new_comm, new_size, ierr)
+END SUBROUTINE split_world_odd_even
+
+SUBROUTINE split_world_only_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = MPI_UNDEFINED
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  if(new_comm /= MPI_COMM_NULL) then
+    call MPI_Comm_rank(new_comm, new_rank, ierr)
+    call MPI_Comm_size(new_comm, new_size, ierr)
+  end if
+END SUBROUTINE split_world_only_even
+
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Check the contents of file : Check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+    character(len=PIO_TF_MAX_STR_LEN) :: val
+
+    ret = PIO_inq_dimid(pio_file, dimname, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_inq_varid(pio_file, attname, pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_get_att(pio_file, pio_var, attname, val)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to get att " // trim(attname) // " in file :" // trim(fname))
+
+    PRINT *, "val = ", trim(val), ", fname =", trim(fname)
+    PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
+END SUBROUTINE check_file
+
+! Open and check the contents of file : open it and check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
+                      attname, dimname, disable_fclose, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    logical, intent(in) :: disable_fclose
+    integer, intent(inout) :: ret
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    call check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
+
+    if(.not. disable_fclose) then
+      call PIO_closefile(pio_file)
+    end if
+END SUBROUTINE open_and_check_file
+
+! Create three files with one iosystem - with all procs, and open/read with
+! another iosystem - subset (odd/even) of procs
+PIO_TF_AUTO_TEST_SUB_BEGIN three_files_two_iosystems_odd_even
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname0 = "pio_iosys_test_file0.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, num_iotypes = 0
+  type(file_desc_t) :: pio_file0, pio_file
+
+  type(iosystem_desc_t) :: odd_even_iosys
+  integer :: odd_even_comm, odd_even_comm_rank, odd_even_comm_size
+  logical :: is_even
+  integer :: ret
+
+  ! Split world to odd and even procs
+  call split_world_odd_even(odd_even_comm, odd_even_comm_rank, odd_even_comm_size, is_even)
+
+  call PIO_init(odd_even_comm_rank, odd_even_comm, odd_even_comm_size, &
+                1, &! Num aggregators
+                1, &! Stride
+                PIO_rearr_subset, odd_even_iosys, base=0)
+  call PIO_seterrorhandling(odd_even_iosys, PIO_BCAST_ERROR)
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create three files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname0, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname0)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file0 from all procs - disable close
+    call open_and_check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                    pio_file0, fname0, attname, dimname, .true., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname0)
+
+    ! Open file1 from odd processes and file2 from even processes
+    ! - disable close
+    if(is_even) then
+      fname => fname1
+    else
+      fname => fname2
+    end if
+
+    call open_and_check_file(odd_even_comm, odd_even_iosys, iotypes(i), &
+                    pio_file, fname, attname, dimname, .true., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname)
+
+    ! Make sure that we can still check the contents of the file
+    call check_file(odd_even_comm, odd_even_iosys, iotypes(i), pio_file, &
+                    fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Checking (second) contents of file failed :" // fname)
+
+    call PIO_closefile(pio_file)
+    call PIO_closefile(pio_file0)
+  end do
+
+  call PIO_finalize(odd_even_iosys, ret)
+  call MPI_Comm_free(odd_even_comm, ret)
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END three_files_two_iosystems_odd_even
diff --git a/src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in b/src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in
new file mode 100644
index 00000000000..66c25690e1e
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in
@@ -0,0 +1,284 @@
+! Split comm world into two comms (even procs and odd procs) and
+! rank == overlapped_rank included in both comms
+SUBROUTINE split_world_two_with_overlap(new_comms, new_ranks, new_sizes, overlapped_rank)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, parameter :: NUM_COMMS = 2
+  integer, dimension(NUM_COMMS), intent(inout) :: new_comms
+  integer, dimension(NUM_COMMS), intent(inout) :: new_ranks
+  integer, dimension(NUM_COMMS), intent(inout) :: new_sizes
+  integer, intent(in) :: overlapped_rank
+
+  integer :: i, ierr
+  integer :: world_group
+  ! first group range (first rank, last rank, stride) for including rank=overlapped_rank
+  ! second group range (first rank, last rank, stride) for other procs
+  !   strided depending on number of comms
+  ! Note: NUM_GROUP_RANGES is always 2, indep of value of NUM_COMMS
+  integer, parameter :: NUM_GROUP_RANGES = 2
+  integer :: nranges
+  integer, dimension(3,NUM_GROUP_RANGES) :: new_group_ranges
+  integer, dimension(NUM_COMMS) :: new_groups
+
+  do i=1,NUM_COMMS
+    new_comms(i) = MPI_COMM_NULL
+    new_ranks(i) = -1
+    new_sizes(i) = 0
+  end do
+
+  call MPI_Comm_group(pio_tf_comm_, world_group, ierr)
+
+  do i=1,NUM_COMMS
+    if(pio_tf_world_sz_ == 1) then
+      nranges = 1
+      new_group_ranges(1,1) = overlapped_rank
+      new_group_ranges(2,1) = overlapped_rank
+      new_group_ranges(3,1) = 1
+    else if(overlapped_rank / NUM_COMMS /= i-1) then
+      nranges = 2
+      ! Include rank == overlapped_rank
+      new_group_ranges(1,1) = overlapped_rank
+      new_group_ranges(2,1) = overlapped_rank
+      new_group_ranges(3,1) = 1
+
+      ! Include other processes split evenly among NUM_COMMS
+      new_group_ranges(1,2) = i-1
+      new_group_ranges(2,2) = i-1 + (pio_tf_world_sz_/NUM_COMMS - 1) * NUM_COMMS
+      new_group_ranges(3,2) = NUM_COMMS
+    else
+      nranges = 1
+      ! Include processes split evenly among NUM_COMMS
+      ! rank == overlapped_rank is already included in this range
+      new_group_ranges(1,1) = i-1
+      new_group_ranges(2,1) = i-1 + (pio_tf_world_sz_/NUM_COMMS - 1) * NUM_COMMS
+      new_group_ranges(3,1) = NUM_COMMS
+    end if
+
+    call MPI_Group_range_incl(world_group, nranges, new_group_ranges,&
+          new_groups(i), ierr)
+  end do
+
+  call MPI_Group_free(world_group, ierr)
+
+  ! Create communicators corresponding to the groups
+  ! All the communicators will have rank == overlapped_rank and are
+  ! disjoint otherwise
+  do i=1,NUM_COMMS
+    call MPI_Comm_create(pio_tf_comm_, new_groups(i), new_comms(i), ierr)
+    call MPI_Group_free(new_groups(i), ierr)
+    if(new_comms(i) /= MPI_COMM_NULL) then
+      call MPI_Comm_rank(new_comms(i), new_ranks(i), ierr)
+      call MPI_Comm_size(new_comms(i), new_sizes(i), ierr)
+    else
+      new_ranks(i) = -1
+      new_sizes(i) = 0
+    end if
+  end do
+
+END SUBROUTINE split_world_two_with_overlap
+
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Check the contents of file : Check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+    character(len=PIO_TF_MAX_STR_LEN) :: val
+
+    ret = PIO_inq_dimid(pio_file, dimname, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_inq_varid(pio_file, attname, pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_get_att(pio_file, pio_var, attname, val)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to get att " // trim(attname) // " in file :" // trim(fname))
+
+    PRINT *, "val = ", trim(val), ", fname =", trim(fname)
+    PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
+END SUBROUTINE check_file
+
+! Open and check the contents of file : open it and check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
+                      attname, dimname, disable_fclose, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    logical, intent(in) :: disable_fclose
+    integer, intent(inout) :: ret
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    call check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
+
+    if(.not. disable_fclose) then
+      call PIO_closefile(pio_file)
+    end if
+END SUBROUTINE open_and_check_file
+
+! Create three files with one iosystem - with all procs, and open/read with
+! two different iosystems - subset (odd/even) of procs
+! The two iosystems created overlap at rank=0 (and are disjoint otherwise)
+PIO_TF_AUTO_TEST_SUB_BEGIN three_files_two_iosystems_with_overlap
+  use mpi
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname0 = "pio_iosys_test_file0.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, j, num_iotypes = 0
+  type(file_desc_t) :: pio_file0
+  integer, parameter :: NUM_COMMS = 2
+  type(file_desc_t), dimension(NUM_COMMS) :: pio_files
+  integer :: pio_base_rank = 0, pio_num_iotasks = 0
+
+  type(iosystem_desc_t), dimension(NUM_COMMS) :: overlapped_iosys
+  integer, dimension(NUM_COMMS) :: overlapped_comms, overlapped_comm_ranks, overlapped_comm_sizes
+  integer :: ret
+
+  ! Split world to two disjoint comms with overlap only at overlapped_rank=0
+  call split_world_two_with_overlap(overlapped_comms, overlapped_comm_ranks, overlapped_comm_sizes, 0)
+
+  do i=1,NUM_COMMS
+    if(overlapped_comms(i) /= MPI_COMM_NULL) then
+      ! If we have more than 1 proc, make sure that the io tasks start from 1
+      ! since rank=0 is always shared by all the overlapped_comms above
+      if(overlapped_comm_sizes(i) > 1) then
+        pio_base_rank = 1
+        pio_num_iotasks = overlapped_comm_sizes(i) - 1
+      else
+        pio_base_rank = 0
+        pio_num_iotasks = overlapped_comm_sizes(i)
+      end if
+      call PIO_init(overlapped_comm_ranks(i), overlapped_comms(i), &
+                    pio_num_iotasks, &
+                    1, &! Num aggregators
+                    1, &! Stride
+                    PIO_rearr_subset, overlapped_iosys(i), base=pio_base_rank)
+      call PIO_seterrorhandling(overlapped_iosys(i), PIO_BCAST_ERROR)
+    end if
+  end do
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create three files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname0, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname0)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file0 from all procs - disable close
+    call open_and_check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                    pio_file0, fname0, attname, dimname, .true., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname0)
+
+    fname => fname1
+
+    do j=1,NUM_COMMS
+      ! The two comms operate on different files
+      if(fname == fname1) then
+        fname => fname2
+      else
+        fname => fname1
+      end if
+      if(overlapped_comms(j) /= MPI_COMM_NULL) then
+        call open_and_check_file(overlapped_comms(j), overlapped_iosys(j), iotypes(i), &
+                        pio_files(j), fname, attname, dimname, .true., ret)
+        PIO_TF_CHECK_ERR(ret, overlapped_comms(j), "Checking contents of file failed :" // fname)
+
+        ! Make sure that we can still check the contents of the file
+        call check_file(overlapped_comms(j), overlapped_iosys(j), iotypes(i), &
+                        pio_files(j), fname, attname, dimname, ret)
+        PIO_TF_CHECK_ERR(ret, overlapped_comms(j), "Checking (second) contents of file failed :" // fname)
+      end if
+    end do
+
+    do j=1,NUM_COMMS
+      if(overlapped_comms(j) /= MPI_COMM_NULL) then
+        call PIO_closefile(pio_files(j))
+      end if
+    end do
+    call PIO_closefile(pio_file0)
+  end do
+
+  do i=1,NUM_COMMS
+    if(overlapped_comms(i) /= MPI_COMM_NULL) then
+      call PIO_finalize(overlapped_iosys(i), ret)
+      call MPI_Comm_free(overlapped_comms(i), ret)
+    end if
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END three_files_two_iosystems_with_overlap
diff --git a/src/externals/pio2/tests/general/pio_rearr.F90.in b/src/externals/pio2/tests/general/pio_rearr.F90.in
new file mode 100644
index 00000000000..f3a28c4e128
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_rearr.F90.in
@@ -0,0 +1,252 @@
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to enddef, file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Open a file and perform read and write
+SUBROUTINE open_and_check_rdwr(comm, iosys, iotype, pio_file, fname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    integer, intent(inout) :: ret
+
+    integer, parameter :: VEC_LOCAL_SZ = 3
+    character(len=PIO_TF_MAX_STR_LEN) :: var_name = "test_1d_var"
+    character(len=PIO_TF_MAX_STR_LEN) :: dim_name = "test_dim_1d_var"
+    type(var_desc_t) :: pio_var
+    type(io_desc_t) :: iodesc
+    integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+    integer :: start_compdof
+    real, dimension(VEC_LOCAL_SZ) :: wbuf
+    real, dimension(:), allocatable :: rbuf
+    integer, dimension(1) :: dims
+    integer :: pio_dim
+    integer :: comm_rank, comm_sz
+    integer :: i
+
+    call MPI_Comm_size(comm, comm_sz, ret)
+    call MPI_Comm_rank(comm, comm_rank, ret)
+
+    dims(1) = VEC_LOCAL_SZ * comm_sz
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    ret = pio_inq_dimid(pio_file, dim_name, pio_dim)
+    if(ret /= PIO_NOERR) then
+      ! Define the required dim/var
+      ret = PIO_redef(pio_file)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to redef:" // fname)
+
+      ret = PIO_def_dim(pio_file, dim_name, dims(1), pio_dim)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim:" // fname)
+
+      ! Assume var is not defined either
+      ret = PIO_def_var(pio_file, var_name, pio_real, (/pio_dim/), pio_var)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to define var:" // fname)
+
+      ret = PIO_enddef(pio_file)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to enddef:" // fname)
+    else
+      ! Assume var was also defined
+      ret = PIO_inq_varid(pio_file, var_name, pio_var)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to inq var:" // fname)
+    end if
+
+    ! Compdof is a simple reverse (of chunks of VEC_LOCAL_SZ) of the file offsets
+    ! e.g. VEC_LOCAL_SZ = 3, with 2 procs
+    ! [4 5 6] [1 2 3]
+    start_compdof = (comm_sz - comm_rank - 1) * VEC_LOCAL_SZ
+    do i=1,VEC_LOCAL_SZ
+      compdof_rel_disps(i) = i
+    end do
+
+    compdof = start_compdof + compdof_rel_disps
+    wbuf = compdof
+    allocate(rbuf(size(wbuf)))
+    rbuf = 0
+
+    call PIO_initdecomp(iosys, pio_real, dims, compdof, iodesc)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to init decomp:" // fname)
+
+    ! Write and read back the data
+    call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to write darray: " // fname)
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to read darray: " // fname)
+
+    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+    deallocate(rbuf)
+
+    call PIO_freedecomp(iosys, iodesc)
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE open_and_check_rdwr
+
+PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_base
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname = "pio_test_rearrs_base.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, j, k, num_iotypes = 0
+  type(file_desc_t) :: pio_file
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: dup_comm_rank, dup_comm_sz, dup_iosys_base
+  integer :: ret
+
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing iotype: ", iotype_descs(i)
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, pio_tf_comm_, "Failed to create file :" // fname)
+
+    call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+    call MPI_Comm_rank(dup_comm, dup_comm_rank, ret)
+    call MPI_Comm_size(dup_comm, dup_comm_sz, ret)
+    do j=1, NUM_REARRANGERS
+      PIO_TF_LOG(0,*) "Testing rearr : ", rearrs_info(j)
+      do k=1, dup_comm_sz
+        dup_iosys_base = k-1
+        PIO_TF_LOG(0,*) "Base = ", dup_iosys_base
+        call PIO_init(pio_tf_world_rank_, dup_comm, pio_tf_world_sz_, &
+                      1, &! Num aggregators
+                      1, &! Stride
+                      rearrs(j), dup_iosys, base=dup_iosys_base)
+        call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+        call open_and_check_rdwr(dup_comm, dup_iosys, iotypes(i), &
+                        pio_file, fname, ret)
+        PIO_TF_CHECK_ERR(ret, dup_comm, "Checking rd+wr on file failed :" // fname)
+
+        call PIO_finalize(dup_iosys, ret)
+      end do
+    end do
+    call PIO_deletefile(pio_tf_iosystem_, fname)
+    call MPI_Comm_free(dup_comm, ret)
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END test_rearrs_base
+
+! Test different combinations of the rearrangers to test the compatibility
+! between different rearrangers
+! init/rd+wr/finalize with one rearranger followed by another and try
+! all combinations ((subset,box),(box_subset),(box,box),(subset,subset))
+PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_combs
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname = "pio_rearrs_combs.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  integer, parameter :: MAX_PERMS = 4
+  integer :: rearrs_perms(NUM_REARRANGERS,MAX_PERMS) = reshape(&
+                (/pio_rearr_subset, pio_rearr_box,&
+                  pio_rearr_box, pio_rearr_subset,&
+                  pio_rearr_subset, pio_rearr_subset,&
+                  pio_rearr_box, pio_rearr_box/),&
+                  (/NUM_REARRANGERS,MAX_PERMS/)&
+              ) 
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_perms_info(NUM_REARRANGERS,MAX_PERMS) =&
+            reshape(&
+              (/"PIO_REARR_SUBSET", "PIO_REARR_BOX   ",&
+              "PIO_REARR_BOX   ", "PIO_REARR_SUBSET",&
+              "PIO_REARR_SUBSET", "PIO_REARR_SUBSET",&
+              "PIO_REARR_BOX   ", "PIO_REARR_BOX   "/),&
+              (/NUM_REARRANGERS,MAX_PERMS/)&
+            )
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, j, k, num_iotypes = 0
+  type(file_desc_t) :: pio_file
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: ret
+
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing iotype: ", iotype_descs(i)
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, pio_tf_comm_, "Failed to create file :" // fname)
+
+    call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+    ! Try different combinations of rearrangers
+    do k=1,MAX_PERMS
+      do j=1, NUM_REARRANGERS
+        PIO_TF_LOG(0,*) "Testing rearr : ", rearrs_perms_info(j,k)
+        call PIO_init(pio_tf_world_rank_, dup_comm, pio_tf_world_sz_, &
+                      1, &! Num aggregators
+                      1, &! Stride
+                      rearrs_perms(j,k), dup_iosys, base=0)
+        call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+        call open_and_check_rdwr(dup_comm, dup_iosys, iotypes(i), &
+                        pio_file, fname, ret)
+        PIO_TF_CHECK_ERR(ret, dup_comm, "Checking rd+wr on file failed :" // fname)
+
+        call PIO_finalize(dup_iosys, ret)
+      end do
+    end do
+    call PIO_deletefile(pio_tf_iosystem_, fname)
+    call MPI_Comm_free(dup_comm, ret)
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END test_rearrs_combs
+
diff --git a/src/externals/pio2/tests/general/pio_rearr_opts.F90.in b/src/externals/pio2/tests/general/pio_rearr_opts.F90.in
new file mode 100644
index 00000000000..9d168ea0473
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_rearr_opts.F90.in
@@ -0,0 +1,483 @@
+MODULE pio_rearr_opts_tgv
+  use pio_tutil
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_fname ="pio_rearr_opts_test.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_var_name ="dummy_var"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_dim_name ="dummy_dim"
+END MODULE pio_rearr_opts_tgv
+
+! Test init/finalize wiht a default set of rearranger options
+PIO_TF_AUTO_TEST_SUB_BEGIN init_fin_with_rearr_opts
+  implicit none
+
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  type(pio_rearr_opt_t) :: pio_rearr_opts
+  ! Dummy val for max pend req
+  integer, parameter :: MAX_PEND_REQ = 10
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: i, ret
+
+  call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+
+  do i=1,NUM_REARRANGERS
+    ! Some dummy values for rearranger options
+    pio_rearr_opts%comm_type = PIO_rearr_comm_p2p
+    pio_rearr_opts%fcd = PIO_rearr_comm_fc_2d_enable
+
+    pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = PIO_REARR_COMM_UNLIMITED_PEND_REQ
+    pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .true.
+    pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .true.
+
+    pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req = MAX_PEND_REQ
+    pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .true.
+    pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .true.
+
+    call PIO_init(pio_tf_world_rank_, dup_comm, pio_tf_world_sz_, &
+                  1, &! Num aggregators
+                  1, &! Stride
+                  rearrs(i), dup_iosys, base=0, rearr_opts=pio_rearr_opts)
+    call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+    call PIO_finalize(dup_iosys, ret)
+    PIO_TF_CHECK_ERR(ret, "Finalize failed")
+  end do
+
+  call MPI_Comm_free(dup_comm, ret)
+
+PIO_TF_AUTO_TEST_SUB_END init_fin_with_rearr_opts
+
+SUBROUTINE print_rearr_opts(pio_rearr_opts, ret)
+  use pio_tutil
+  implicit none
+
+  type(pio_rearr_opt_t), intent(in) :: pio_rearr_opts
+  integer, intent(inout) :: ret
+
+  ret = PIO_NOERR
+
+  PIO_TF_LOG(0,*) "PIO rearranger options :"
+  if(pio_rearr_opts%comm_type == PIO_rearr_comm_p2p) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_p2p"
+  else if(pio_rearr_opts%comm_type == PIO_rearr_comm_coll) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_coll"
+  else
+    PIO_TF_LOG(0,*) " comm_type = INVALID"
+  end if
+
+  if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_enable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_enable"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_comp2io"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_io2comp"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_disable"
+  else
+    PIO_TF_LOG(0,*) " fcd = INVALID"
+  end if
+
+  PIO_TF_LOG(0,*) " comp2io max_pend_req =", pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_hs) then
+    PIO_TF_LOG(0,*) " comp2io enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_isend) then
+    PIO_TF_LOG(0,*) " comp2io enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_isend = FALSE"
+  end if
+
+  PIO_TF_LOG(0,*) " io2comp max_pend_req =", pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_hs) then
+    PIO_TF_LOG(0,*) " io2comp enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_isend) then
+    PIO_TF_LOG(0,*) " io2comp enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_isend = FALSE"
+  end if
+
+END SUBROUTINE print_rearr_opts
+
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ elements
+! # All odd procs have VEC_LOCAL_SZ + 1 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4,5] [6,7]
+! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
+! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [3,4,5] [1,2] [6,7]
+! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
+! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
+SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
+  implicit none
+
+  integer, parameter :: VEC_LOCAL_SZ = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_LOCAL_SZ + 1
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
+      else
+        count(1) = VEC_LOCAL_SZ
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_LOCAL_SZ
+      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_LOCAL_SZ
+    else
+      count(1) = VEC_LOCAL_SZ + 1
+    end if
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Create a decomp that will be used by tests below
+! The iodesc needs to be freed by the caller
+SUBROUTINE create_decomp_and_init_buf(iosys, iocomm, iodesc, wbuf, dims, ret)
+  use pio_tutil
+  implicit none
+
+  type(iosystem_desc_t), intent(inout) :: iosys
+  integer, intent(in) :: iocomm
+  type(io_desc_t), intent(out) :: iodesc
+  real, dimension(:), allocatable, intent(inout) :: wbuf
+  integer, dimension(1), intent(out) :: dims
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  integer :: i
+  integer :: rank, sz
+
+  ret = PIO_NOERR
+
+  call MPI_Comm_size(iocomm, sz, ret)
+  call MPI_Comm_rank(iocomm, rank, ret)
+
+  call get_1d_bc_info(rank, sz, dims, start, count, .true.)
+  allocate(wbuf(count(1)))
+  allocate(compdof(count(1)))
+  do i=1,count(1)
+    wbuf(i) = start(1) + i - 1
+    compdof(i) = wbuf(i)
+  end do
+
+  call PIO_initdecomp(iosys, PIO_real, dims, compdof, iodesc)
+  deallocate(compdof)
+
+END SUBROUTINE
+
+! Create file and var used in the tests below
+! All details are picked from pio_rearr_opts_tgv module
+SUBROUTINE create_file_and_var(iotype, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  integer, intent(in) :: iotype
+  integer, intent(out) :: ret
+
+  type(file_desc_t) :: pio_file
+  type(var_desc_t) :: pio_var
+  integer :: pio_dim
+  integer, dimension(1) :: dims
+  integer, dimension(1) :: start, count
+
+  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotype, tgv_fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_def_dim(pio_file, tgv_dim_name, dims(1), pio_dim)
+  PIO_TF_CHECK_ERR(ret, "Failed to define dim file =" // trim(tgv_fname))
+
+  ret = PIO_def_var(pio_file, tgv_var_name, pio_real, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var file =" // trim(tgv_fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef file =" // trim(tgv_fname))
+
+  call PIO_closefile(pio_file)
+
+END SUBROUTINE
+
+! Open file and inq var
+! All details are picked from pio_rearr_opts_tgv module
+! Note: The file is kept open so the called needs to close it
+SUBROUTINE open_file_and_get_var(iosys, pio_file, iotype, pio_var, dims, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  type(iosystem_desc_t), intent(inout) :: iosys
+  type(file_desc_t), intent(out) :: pio_file
+  integer, intent(in) :: iotype
+  type(var_desc_t), intent(out) :: pio_var
+  integer, dimension(1), intent(in) :: dims
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+
+  ret = PIO_openfile(iosys, pio_file, iotype, tgv_fname, pio_write)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_inq_varid(pio_file, tgv_var_name, pio_var)
+  PIO_TF_CHECK_ERR(ret, "Could not inq var " // trim(tgv_fname))
+
+END SUBROUTINE
+
+
+! Test all possible combinations of rearranger options
+PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts
+  use pio_rearr_opts_tgv
+  implicit none
+  interface
+    subroutine create_decomp_and_init_buf(iosys, iocomm, iodesc, wbuf, dims, ret)
+      use pio_tutil
+      type(iosystem_desc_t), intent(inout) :: iosys
+      integer, intent(in) :: iocomm
+      type(io_desc_t), intent(out) :: iodesc
+      real, dimension(:), allocatable, intent(inout) :: wbuf
+      integer, dimension(1), intent(out) :: dims
+      integer, intent(out) :: ret
+    end subroutine create_decomp_and_init_buf
+  end interface 
+
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  type(pio_rearr_opt_t) :: pio_rearr_opts
+
+  ! Different rearranger options that are tested here
+  integer, parameter :: NUM_COMM_TYPE_OPTS = 2
+  integer :: comm_type_opts(NUM_COMM_TYPE_OPTS) =&
+                  (/pio_rearr_comm_p2p,pio_rearr_comm_coll/)
+  integer, parameter :: NUM_FCD_OPTS = 4
+  integer :: fcd_opts(NUM_FCD_OPTS) = &
+                  (/pio_rearr_comm_fc_2d_disable,&
+                    pio_rearr_comm_fc_1d_comp2io,&
+                    pio_rearr_comm_fc_1d_io2comp,&
+                    pio_rearr_comm_fc_2d_enable/)
+  integer :: num_fcd_opts_comm_type
+  integer, parameter :: NUM_ENABLE_HS_OPTS = 2
+  logical :: enable_hs_opts(NUM_ENABLE_HS_OPTS) = (/.true.,.false./)
+  integer :: num_enable_hs_opts_comp2io, num_enable_hs_opts_io2comp
+  integer, parameter :: NUM_ENABLE_ISEND_OPTS = 2
+  logical :: enable_isend_opts(NUM_ENABLE_ISEND_OPTS) = (/.true.,.false./)
+  integer :: num_enable_isend_opts_comp2io, num_enable_isend_opts_io2comp
+  integer, parameter :: NUM_MAX_PEND_REQ_OPTS = 2
+  integer :: max_pend_req_opts(NUM_MAX_PEND_REQ_OPTS) = &
+                  (/pio_rearr_comm_unlimited_pend_req, 2/)
+  integer :: num_max_pend_req_opts_comp2io, num_max_pend_req_opts_io2comp
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: cur_rearr, cur_comm_type_opt, cur_fcd_opt, cur_enable_hs_c2i, &
+              cur_enable_isend_c2i, cur_max_pend_req_c2i, &
+              cur_enable_hs_i2c, cur_enable_isend_i2c, cur_max_pend_req_i2c
+
+  type(file_desc_t) :: pio_file
+  type(io_desc_t) :: iodesc
+  type(var_desc_t) :: pio_var
+  integer, dimension(1) :: dims
+
+  real, dimension(:), allocatable :: rbuf, wbuf
+
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: ret, ierr, i
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create the file and decomp 
+    call create_file_and_var(iotypes(i), ret)
+    PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+    call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+
+    ! Test all combinations of these flow control parameters
+    do cur_rearr=1,NUM_REARRANGERS
+      PIO_TF_LOG(0,*) "Testing ", rearrs_info(cur_rearr)
+      do cur_comm_type_opt=1,NUM_COMM_TYPE_OPTS
+        pio_rearr_opts%comm_type = comm_type_opts(cur_comm_type_opt)
+        if(pio_rearr_opts%comm_type == pio_rearr_comm_coll) then
+          ! For coll we only test pio_rearr_comm_fc_2d_disable
+          num_fcd_opts_comm_type = 1
+        else if(pio_rearr_opts%comm_type == pio_rearr_comm_p2p) then
+          ! for p2p we test all possible combinations
+          num_fcd_opts_comm_type = NUM_FCD_OPTS
+        else
+          PIO_TF_ERROR("Unexpected comm type")
+        end if
+        do cur_fcd_opt=1,num_fcd_opts_comm_type
+          pio_rearr_opts%fcd = fcd_opts(cur_fcd_opt)
+          if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_enable) then
+            num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+            num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+            num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+            num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+            num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+            num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+          else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_comp2io) then
+            ! Only test different rearr opts in comp2io direction
+            num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+            num_enable_hs_opts_io2comp = 1
+            num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+            num_enable_isend_opts_io2comp = 1
+            num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+            num_max_pend_req_opts_io2comp = 1
+          else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_io2comp) then
+            ! Only test different rearr opts in io2comp direction
+            num_enable_hs_opts_comp2io = 1
+            num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+            num_enable_isend_opts_comp2io = 1
+            num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+            num_max_pend_req_opts_comp2io = 1
+            num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+          else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_disable) then
+            ! Just test one default combination
+            num_enable_hs_opts_comp2io = 1
+            num_enable_hs_opts_io2comp = 1
+            num_enable_isend_opts_comp2io = 1
+            num_enable_isend_opts_io2comp = 1
+            num_max_pend_req_opts_comp2io = 1
+            num_max_pend_req_opts_io2comp = 1
+          else
+            PIO_TF_ERROR("Unexpected flow control option")
+          end if
+          do cur_enable_hs_c2i=1,num_enable_hs_opts_comp2io
+            pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = &
+                  enable_hs_opts(cur_enable_hs_c2i)
+
+            do cur_enable_isend_c2i=1,num_enable_isend_opts_comp2io
+              pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = &
+                    enable_isend_opts(cur_enable_isend_c2i)
+
+              do cur_max_pend_req_c2i=1,num_max_pend_req_opts_comp2io
+                pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = &
+                    max_pend_req_opts(cur_max_pend_req_c2i)
+
+                do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp
+                  pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =&
+                    enable_hs_opts(cur_enable_hs_i2c)
+                
+                  do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp
+                    pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =&
+                      enable_isend_opts(cur_enable_isend_i2c)
+
+                    do cur_max_pend_req_i2c=1,num_max_pend_req_opts_io2comp
+                      pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req =&
+                        max_pend_req_opts(cur_max_pend_req_i2c)
+
+                      call print_rearr_opts(pio_rearr_opts, ret)
+                      ! Ignoring return value - just printing for info
+                      call PIO_init(pio_tf_world_rank_,&
+                                      dup_comm, pio_tf_world_sz_, &
+                                      1, &! Num aggregators
+                                      1, &! Stride
+                                      rearrs(cur_rearr),&
+                                      dup_iosys, base=0,&
+                                      rearr_opts=pio_rearr_opts)
+
+                      call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+                      call create_decomp_and_init_buf(dup_iosys, dup_comm, iodesc, wbuf, dims, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Creating decomp failed")
+
+                      allocate(rbuf(size(wbuf)))
+                      rbuf = 0
+
+                      call open_file_and_get_var(dup_iosys, pio_file, iotypes(i),&
+                              pio_var, dims, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+                      call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Writing var failed fname="//trim(tgv_fname))
+
+                      call PIO_syncfile(pio_file)
+
+                      call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Reading var failed fname="//trim(tgv_fname))
+
+                      PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+                      call PIO_freedecomp(dup_iosys, iodesc)
+
+                      deallocate(rbuf)
+                      if(allocated(wbuf)) then
+                        deallocate(wbuf)
+                      end if
+
+                      call PIO_closefile(pio_file)
+
+                      call PIO_finalize(dup_iosys, ret)
+                      PIO_TF_CHECK_ERR(ret, "Finalize failed")
+                    end do ! cur_max_pend_req_i2c
+                  end do ! cur_enable_isend_i2c
+                end do ! cur_enable_hs_i2c
+              end do ! cur_max_pend_req_c2i    
+            end do ! cur_enable_isend_c2i
+          end do ! cur_enable_hs_c2i
+        end do ! cur_fcd_opt
+      end do ! cur_comm_type_opt
+    end do ! cur_rearr
+
+    call PIO_deletefile(pio_tf_iosystem_, trim(tgv_fname))
+
+    call MPI_Comm_free(dup_comm, ret)
+  end do ! iotypes
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END write_with_rearr_opts
diff --git a/src/externals/pio2/tests/general/pio_rearr_opts2.F90.in b/src/externals/pio2/tests/general/pio_rearr_opts2.F90.in
new file mode 100644
index 00000000000..9744a87cfc3
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_rearr_opts2.F90.in
@@ -0,0 +1,420 @@
+MODULE pio_rearr_opts_tgv
+  use pio_tutil
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_fname ="pio_rearr_opts2_test.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_var_name ="dummy_var"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_dim_name ="dummy_dim"
+END MODULE pio_rearr_opts_tgv
+
+SUBROUTINE print_rearr_opts(pio_rearr_opts, ret)
+  use pio_tutil
+  implicit none
+
+  type(pio_rearr_opt_t), intent(in) :: pio_rearr_opts
+  integer, intent(inout) :: ret
+
+  ret = PIO_NOERR
+
+  PIO_TF_LOG(0,*) "PIO rearranger options :"
+  if(pio_rearr_opts%comm_type == PIO_rearr_comm_p2p) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_p2p"
+  else if(pio_rearr_opts%comm_type == PIO_rearr_comm_coll) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_coll"
+  else
+    PIO_TF_LOG(0,*) " comm_type = INVALID"
+  end if
+
+  if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_enable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_enable"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_comp2io"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_io2comp"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_disable"
+  else
+    PIO_TF_LOG(0,*) " fcd = INVALID"
+  end if
+
+  PIO_TF_LOG(0,*) " comp2io max_pend_req =", pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_hs) then
+    PIO_TF_LOG(0,*) " comp2io enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_isend) then
+    PIO_TF_LOG(0,*) " comp2io enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_isend = FALSE"
+  end if
+
+  PIO_TF_LOG(0,*) " io2comp max_pend_req =", pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_hs) then
+    PIO_TF_LOG(0,*) " io2comp enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_isend) then
+    PIO_TF_LOG(0,*) " io2comp enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_isend = FALSE"
+  end if
+
+END SUBROUTINE print_rearr_opts
+
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ elements
+! # All odd procs have VEC_LOCAL_SZ + 1 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4,5] [6,7]
+! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
+! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [3,4,5] [1,2] [6,7]
+! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
+! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
+SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
+  implicit none
+
+  integer, parameter :: VEC_LOCAL_SZ = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_LOCAL_SZ + 1
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
+      else
+        count(1) = VEC_LOCAL_SZ
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_LOCAL_SZ
+      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_LOCAL_SZ
+    else
+      count(1) = VEC_LOCAL_SZ + 1
+    end if
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Create a decomp that will be used by tests below
+! The iodesc needs to be freed by the caller
+SUBROUTINE create_decomp_and_init_buf(iodesc, wbuf, dims, ret)
+  use pio_tutil
+  implicit none
+
+  type(io_desc_t), intent(out) :: iodesc
+  real, dimension(:), allocatable, intent(inout) :: wbuf
+  integer, dimension(1), intent(out) :: dims
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  integer :: i
+
+  ret = PIO_NOERR
+
+  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  allocate(wbuf(count(1)))
+  allocate(compdof(count(1)))
+  do i=1,count(1)
+    wbuf(i) = start(1) + i - 1
+    compdof(i) = wbuf(i)
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_real, dims, compdof, iodesc)
+  deallocate(compdof)
+
+END SUBROUTINE
+
+! Create file and var used in the tests below
+! All details are picked from pio_rearr_opts_tgv module
+SUBROUTINE create_file_and_var(iotype, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  integer, intent(in) :: iotype
+  integer, intent(out) :: ret
+
+  type(file_desc_t) :: pio_file
+  type(var_desc_t) :: pio_var
+  integer :: pio_dim
+  integer, dimension(1) :: dims
+  integer, dimension(1) :: start, count
+
+  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotype, tgv_fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_def_dim(pio_file, tgv_dim_name, dims(1), pio_dim)
+  PIO_TF_CHECK_ERR(ret, "Failed to define dim file =" // trim(tgv_fname))
+
+  ret = PIO_def_var(pio_file, tgv_var_name, pio_real, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var file =" // trim(tgv_fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef file =" // trim(tgv_fname))
+
+  call PIO_closefile(pio_file)
+
+END SUBROUTINE
+
+! Open file and inq var
+! All details are picked from pio_rearr_opts_tgv module
+! Note: The file is kept open so the called needs to close it
+SUBROUTINE open_file_and_get_var(pio_file, iotype, pio_var, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  type(file_desc_t), intent(out) :: pio_file
+  integer, intent(in) :: iotype
+  type(var_desc_t), intent(out) :: pio_var
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+
+  ret = PIO_openfile(pio_tf_iosystem_, pio_file, iotype, tgv_fname, pio_write)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_inq_varid(pio_file, tgv_var_name, pio_var)
+  PIO_TF_CHECK_ERR(ret, "Could not inq var " // trim(tgv_fname))
+
+END SUBROUTINE
+
+! Test all possible combinations of rearranger options
+! using the pio interface to explicitly set the rearranger options
+! after init.
+PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write
+  use pio_rearr_opts_tgv
+  implicit none
+  interface
+    subroutine create_decomp_and_init_buf(iodesc, wbuf, dims, ret)
+      use pio_tutil
+      type(io_desc_t), intent(out) :: iodesc
+      real, dimension(:), allocatable, intent(inout) :: wbuf
+      integer, dimension(1), intent(out) :: dims
+      integer, intent(out) :: ret
+    end subroutine create_decomp_and_init_buf
+  end interface 
+
+  type(pio_rearr_opt_t) :: pio_rearr_opts
+
+  ! Different rearranger options that are tested here
+  integer, parameter :: NUM_COMM_TYPE_OPTS = 2
+  integer :: comm_type_opts(NUM_COMM_TYPE_OPTS) =&
+                  (/pio_rearr_comm_p2p,pio_rearr_comm_coll/)
+  integer, parameter :: NUM_FCD_OPTS = 4
+  integer :: fcd_opts(NUM_FCD_OPTS) = &
+                  (/pio_rearr_comm_fc_2d_disable,&
+                    pio_rearr_comm_fc_1d_comp2io,&
+                    pio_rearr_comm_fc_1d_io2comp,&
+                    pio_rearr_comm_fc_2d_enable/)
+  integer :: num_fcd_opts_comm_type
+  integer, parameter :: NUM_ENABLE_HS_OPTS = 2
+  logical :: enable_hs_opts(NUM_ENABLE_HS_OPTS) = (/.true.,.false./)
+  integer :: num_enable_hs_opts_comp2io, num_enable_hs_opts_io2comp
+  integer, parameter :: NUM_ENABLE_ISEND_OPTS = 2
+  logical :: enable_isend_opts(NUM_ENABLE_ISEND_OPTS) = (/.true.,.false./)
+  integer :: num_enable_isend_opts_comp2io, num_enable_isend_opts_io2comp
+  integer, parameter :: NUM_MAX_PEND_REQ_OPTS = 3
+  integer :: max_pend_req_opts(NUM_MAX_PEND_REQ_OPTS) = &
+                  (/pio_rearr_comm_unlimited_pend_req, 1, 2/)
+  integer :: num_max_pend_req_opts_comp2io, num_max_pend_req_opts_io2comp
+
+  integer ::  cur_comm_type_opt, cur_fcd_opt, cur_enable_hs_c2i, &
+              cur_enable_isend_c2i, cur_max_pend_req_c2i, &
+              cur_enable_hs_i2c, cur_enable_isend_i2c, cur_max_pend_req_i2c
+
+  type(file_desc_t) :: pio_file
+  type(io_desc_t) :: iodesc
+  type(var_desc_t) :: pio_var
+  integer, dimension(1) :: dims
+
+  real, dimension(:), allocatable :: rbuf, wbuf
+
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: ret, ierr, i
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create the file and decomp 
+    call create_file_and_var(iotypes(i), ret)
+    PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+    call open_file_and_get_var(pio_file, iotypes(i), pio_var, ret)
+    PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+    ! Test all combinations of these flow control parameters
+    do cur_comm_type_opt=1,NUM_COMM_TYPE_OPTS
+      pio_rearr_opts%comm_type = comm_type_opts(cur_comm_type_opt)
+      if(pio_rearr_opts%comm_type == pio_rearr_comm_coll) then
+        ! For coll we only test pio_rearr_comm_fc_2d_disable
+        num_fcd_opts_comm_type = 1
+      else if(pio_rearr_opts%comm_type == pio_rearr_comm_p2p) then
+        ! for p2p we test all possible combinations
+        num_fcd_opts_comm_type = NUM_FCD_OPTS
+      else
+        PIO_TF_ERROR("Unexpected comm type")
+      end if
+      do cur_fcd_opt=1,num_fcd_opts_comm_type
+        pio_rearr_opts%fcd = fcd_opts(cur_fcd_opt)
+        if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_enable) then
+          num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+          num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+          num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+          num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+          num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+          num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+        else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_comp2io) then
+          ! Only test different rearr opts in comp2io direction
+          num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+          num_enable_hs_opts_io2comp = 1
+          num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+          num_enable_isend_opts_io2comp = 1
+          num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+          num_max_pend_req_opts_io2comp = 1
+        else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_io2comp) then
+          ! Only test different rearr opts in io2comp direction
+          num_enable_hs_opts_comp2io = 1
+          num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+          num_enable_isend_opts_comp2io = 1
+          num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+          num_max_pend_req_opts_comp2io = 1
+          num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+        else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_disable) then
+          ! Just test one default combination
+          num_enable_hs_opts_comp2io = 1
+          num_enable_hs_opts_io2comp = 1
+          num_enable_isend_opts_comp2io = 1
+          num_enable_isend_opts_io2comp = 1
+          num_max_pend_req_opts_comp2io = 1
+          num_max_pend_req_opts_io2comp = 1
+        else
+          PIO_TF_ERROR("Unexpected flow control option")
+        end if
+        do cur_enable_hs_c2i=1,num_enable_hs_opts_comp2io
+          pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = &
+                enable_hs_opts(cur_enable_hs_c2i)
+
+          do cur_enable_isend_c2i=1,num_enable_isend_opts_comp2io
+            pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = &
+                  enable_isend_opts(cur_enable_isend_c2i)
+
+            do cur_max_pend_req_c2i=1,num_max_pend_req_opts_comp2io
+              pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = &
+                  max_pend_req_opts(cur_max_pend_req_c2i)
+
+              do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp
+                pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =&
+                  enable_hs_opts(cur_enable_hs_i2c)
+              
+                do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp
+                  pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =&
+                    enable_isend_opts(cur_enable_isend_i2c)
+
+                  do cur_max_pend_req_i2c=1,num_max_pend_req_opts_io2comp
+                    pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req =&
+                      max_pend_req_opts(cur_max_pend_req_i2c)
+
+                    call print_rearr_opts(pio_rearr_opts, ret)
+                    ! Ignoring return value - just printing for info
+
+                    ! Set the rearranger options
+                    ! rearr_opt_t descributes logicals as logical(kind=c_bool)
+                    ! but pio_set_rearr_opts() expects regular logicals, hence
+                    ! explicit type cast is reqd for logical params
+                    ret = PIO_set_rearr_opts(pio_tf_iosystem_,&
+                            pio_rearr_opts%comm_type,&
+                            pio_rearr_opts%fcd,&
+                            logical(pio_rearr_opts%comm_fc_opts_comp2io%enable_hs),&
+                            logical(pio_rearr_opts%comm_fc_opts_comp2io%enable_isend),&
+                            pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req,&
+                            logical(pio_rearr_opts%comm_fc_opts_io2comp%enable_hs),&
+                            logical(pio_rearr_opts%comm_fc_opts_io2comp%enable_isend),&
+                            pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req)
+                    PIO_TF_CHECK_ERR(ret, "Setting rearr opts failed fname="//trim(tgv_fname))
+
+                    call create_decomp_and_init_buf(iodesc, wbuf, dims, ret)
+                    PIO_TF_CHECK_ERR(ret, "Creating decomp failed")
+
+                    allocate(rbuf(size(wbuf)))
+                    rbuf = 0
+
+                    call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ret)
+                    PIO_TF_CHECK_ERR(ret, "Writing var failed fname="//trim(tgv_fname))
+
+                    call PIO_syncfile(pio_file)
+
+                    call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ret)
+                    PIO_TF_CHECK_ERR(ret, "Reading var failed fname="//trim(tgv_fname))
+
+                    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+                    call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+
+                    deallocate(rbuf)
+                    if(allocated(wbuf)) then
+                      deallocate(wbuf)
+                    end if
+
+                  end do ! cur_max_pend_req_i2c
+                end do ! cur_enable_isend_i2c
+              end do ! cur_enable_hs_i2c
+            end do ! cur_max_pend_req_c2i    
+          end do ! cur_enable_isend_c2i
+        end do ! cur_enable_hs_c2i
+      end do ! cur_fcd_opt
+    end do ! cur_comm_type_opt
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, trim(tgv_fname))
+  end do ! iotypes
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END set_rearr_opts_and_write
diff --git a/externals/pio2/tests/general/test_memleak.c b/src/externals/pio2/tests/general/test_memleak.c
similarity index 93%
rename from externals/pio2/tests/general/test_memleak.c
rename to src/externals/pio2/tests/general/test_memleak.c
index 8d6b9bce2f8..3c81f477df1 100644
--- a/externals/pio2/tests/general/test_memleak.c
+++ b/src/externals/pio2/tests/general/test_memleak.c
@@ -1,5 +1,5 @@
 /**
- * @file 
+ * @file
  * Tests for handling of fillval.
  *
  * This test was added to track down memory leaks in
@@ -42,19 +42,19 @@
 /** Handle MPI errors. This should only be used with MPI library
  * function calls. */
 #define MPIERR(e) do {                                                  \
-	MPI_Error_string(e, err_buffer, &resultlen);			\
+	MPI_Error_string(e, err_buffer, &resultlen);                    \
 	fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
-	MPI_Finalize();							\
-	return 2;							\
-    } while (0) 
+	MPI_Finalize();                                                 \
+	return 2;                                                       \
+    } while (0)
 
 /** Handle non-MPI errors by finalizing the MPI library and exiting
  * with an exit code. */
-#define ERR(e) do {				\
-        fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \
-	MPI_Finalize();				\
-	return e;				\
-    } while (0) 
+#define ERR(e) do {                             \
+	fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \
+	MPI_Finalize();                         \
+	return e;                               \
+    } while (0)
 
 /** Global err buffer for MPI. When there is an MPI error, this buffer
  * is used to store the error message that is associated with the MPI
@@ -86,7 +86,7 @@ int
 main(int argc, char **argv)
 {
     int verbose = 1;
-    
+
     /** Zero-based rank of processor. */
     int my_rank;
 
@@ -99,7 +99,7 @@ main(int argc, char **argv)
      * classic format file (but with different libraries). The
      * last two produce netCDF4/HDF5 format files, written with
      * and without using netCDF-4 parallel I/O. */
-    int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, 
+    int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF,
 				      PIO_IOTYPE_NETCDF,
 				      PIO_IOTYPE_NETCDF4C,
 				      PIO_IOTYPE_NETCDF4P};
@@ -113,7 +113,7 @@ main(int argc, char **argv)
 							  "test_nc4_classic.nc",
 							  "test_nc4_serial4.nc",
 							  "test_nc4_parallel4.nc"};
-	
+
     /** Number of processors that will do IO. In this example we
      * will do IO from all processors. */
     int niotasks;
@@ -179,13 +179,13 @@ main(int argc, char **argv)
 
     /** Return code. */
     int ret;
-    
-#ifdef TIMING    
+
+#ifdef TIMING
     /* Initialize the GPTL timing library. */
     if ((ret = GPTLinitialize ()))
 	return ret;
-#endif    
-    
+#endif
+
     /* Initialize MPI. */
     if ((ret = MPI_Init(&argc, &argv)))
 	MPIERR(ret);
@@ -206,8 +206,8 @@ main(int argc, char **argv)
 	printf("%d: ParallelIO Library example1 running on %d processors.\n",
 	       my_rank, ntasks);
 
-    /* keep things simple - 1 iotask per MPI process */    
-    niotasks = ntasks; 
+    /* keep things simple - 1 iotask per MPI process */
+    niotasks = ntasks;
 
     /* Initialize the PIO IO system. This specifies how
      * many and which processors are involved in I/O. */
@@ -222,7 +222,7 @@ main(int argc, char **argv)
     for (int i = 0; i < elements_per_pe; i++) {
 	compdof[i] = my_rank * elements_per_pe + i + 1;
     }
-	
+
     /* Create the PIO decomposition for this test. */
     if (verbose)
 	printf("rank: %d Creating decomposition...\n", my_rank);
@@ -236,10 +236,10 @@ main(int argc, char **argv)
     if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init")))
 	MPIERR(ret);
 #endif /* HAVE_MPE */
-	
+
     /* Use PIO to create the example file in each of the four
      * available ways. */
-    for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
+    for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++)
     {
 #ifdef HAVE_MPE
 	/* Log with MPE that we are starting CREATE. */
@@ -254,7 +254,7 @@ main(int argc, char **argv)
 	if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
 				   PIO_CLOBBER)))
 	    ERR(ret);
-	
+
 	/* Define netCDF dimensions and variable. */
 	if (verbose)
 	    printf("rank: %d Defining netCDF metadata...\n", my_rank);
@@ -278,8 +278,8 @@ main(int argc, char **argv)
 	    int storage;
 	    size_t my_chunksize[NDIM];
 	    if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)))
-	    	ERR(ret);
-	    
+		ERR(ret);
+
 	    /** For serial netCDF-4, only processor rank 0 gets the answers. */
 	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank ||
 		format[fmt] == PIO_IOTYPE_NETCDF4P)
@@ -288,7 +288,7 @@ main(int argc, char **argv)
 		    ERR(ERR_AWFUL);
 		for (int d = 0; d < NDIM; d++)
 		    if (my_chunksize[d] != chunksize[d])
-		    	ERR(ERR_AWFUL);
+			ERR(ERR_AWFUL);
 	    }
 
 	    /* Check that the inv_var_deflate functions works. */
@@ -296,7 +296,7 @@ main(int argc, char **argv)
 	    int deflate;
 	    int deflate_level;
 	    if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)))
-	    	ERR(ret);
+		ERR(ret);
 
 	    /** For serial netCDF-4, only processor rank 0 gets the
 	     * answers. Also deflate is turned on by default */
@@ -314,8 +314,8 @@ main(int argc, char **argv)
 	     * in the PIO_ENOTNC4 error. */
 	    if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)) != PIO_ENOTNC4)
 		ERR(ERR_AWFUL);
-	}	    
-	
+	}
+
 	if ((ret = PIOc_enddef(ncid)))
 	    ERR(ret);
 
@@ -325,13 +325,13 @@ main(int argc, char **argv)
 	if ((ret = PIOc_closefile(ncid)))
 	    ERR(ret);
     }
-	
+
     /* Free the PIO decomposition. */
     if (verbose)
 	printf("rank: %d Freeing PIO decomposition...\n", my_rank);
     if ((ret = PIOc_freedecomp(iosysid, ioid)))
 	ERR(ret);
-	
+
     /* Finalize the IO system. */
     if (verbose)
 	printf("rank: %d Freeing PIO resources...\n", my_rank);
@@ -341,11 +341,11 @@ main(int argc, char **argv)
     /* Finalize the MPI library. */
     MPI_Finalize();
 
-#ifdef TIMING    
+#ifdef TIMING
     /* Finalize the GPTL timing library. */
     if ((ret = GPTLfinalize ()))
 	return ret;
-#endif    
-    
+#endif
+
     return 0;
 }
diff --git a/externals/pio2/tests/general/util/pio_tf_f90gen.pl b/src/externals/pio2/tests/general/util/pio_tf_f90gen.pl
similarity index 90%
rename from externals/pio2/tests/general/util/pio_tf_f90gen.pl
rename to src/externals/pio2/tests/general/util/pio_tf_f90gen.pl
index f14b13480ef..f880cda1beb 100755
--- a/externals/pio2/tests/general/util/pio_tf_f90gen.pl
+++ b/src/externals/pio2/tests/general/util/pio_tf_f90gen.pl
@@ -69,7 +69,7 @@ sub generate_gen_templates
   my ($gen_templ_func_bodies) = "";
 
   # Find typenames for the template and corresponding concrete types
-  my (@typenames); 
+  my (@typenames);
   my ($ntypenames);
   my (@typename_counter, @typename_counter_max);
   my ($typename);
@@ -148,7 +148,7 @@ sub generate_gen_templates
 }
 
 # FIXME - Not used, kept around so that the logic is not lost
-# Could be used to generate templates with all permutations of the typenames 
+# Could be used to generate templates with all permutations of the typenames
 sub generate_gen_templates_all_permutes
 {
   my ($gen_templ_func_name, $ref_modif_gen_templ_func_names,
@@ -156,7 +156,7 @@ sub generate_gen_templates_all_permutes
   my ($gen_templ_func_bodies) = "";
 
   # Find typenames for the template and corresponding concrete types
-  my (@typenames); 
+  my (@typenames);
   my ($ntypenames);
   my (@typename_counter, @typename_counter_max);
   my ($typename);
@@ -298,11 +298,11 @@ sub transform_src
       $out_line = $out_line . $1 . "  END IF\n";
       $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
       $out_line = $out_line . $1 . "    IF (pio_tf_retval_utest_ == 0) THEN\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        \"$_\",&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"PASSED\"\n";
       $out_line = $out_line . $1 . "    ELSE\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        \"$_\",&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"FAILED\"\n";
       $out_line = $out_line . $1 . "    END IF\n";
@@ -330,11 +330,11 @@ sub transform_src
       $out_line = $out_line . $1 . "  END IF\n";
       $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
       $out_line = $out_line . $1 . "    IF (pio_tf_retval_utest_ == 0) THEN\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        pio_tf_tmp_log_str_,&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"PASSED\"\n";
       $out_line = $out_line . $1 . "    ELSE\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        pio_tf_tmp_log_str_,&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"FAILED\"\n";
       $out_line = $out_line . $1 . "    END IF\n";
@@ -361,20 +361,34 @@ sub transform_src
     $out_line = $out_line . $1 . "END IF\n";
     $out_line = $out_line . $1 . "IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . $1 . "  IF (pio_tf_retval_utest_ == 0) THEN\n";
-    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . $1 . "      pio_tf_tmp_log_str_,&\n";
     $out_line = $out_line . $1 . "      \"--------\", \"PASSED\"\n";
     $out_line = $out_line . $1 . "  ELSE\n";
-    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . $1 . "      pio_tf_tmp_log_str_,&\n";
     $out_line = $out_line . $1 . "      \"--------\", \"FAILED\"\n";
     $out_line = $out_line . $1 . "  END IF\n";
     $out_line = $out_line . $1 . "END IF";
     $cur_test_case_num += 1;
 }
+  elsif(/^(\s*)PIO_TF_PASSERT\((.+),([^,]+),([^)]+)\)(.*)$/s){
+    $out_line = $1 . $5 . "\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_($2, $3))) THEN\n";
+    $out_line = $out_line . $1 . "  call MPI_COMM_RANK($3, pio_tf_tmp_comm_rank_, pio_tf_retval_utest_)\n";
+    $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
+    $out_line = $out_line . $1 . "  IF (pio_tf_tmp_comm_rank_ == 0) THEN\n";
+    $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: Assertion failed :\",&\n";
+    $out_line = $out_line . $1 . "      " . $4 . ",&\n";
+    $out_line = $out_line . $1 . "       \":\", __FILE__, \":\", __LINE__,&\n";
+    $out_line = $out_line . $1 . "      " . "\"($template_fname:$template_line_no)\"\n";
+    $out_line = $out_line . $1 . "  END IF\n";
+    $out_line = $out_line . $1 . "  RETURN\n";
+    $out_line = $out_line . $1 . "END IF";
+  }
   elsif(/^(\s*)PIO_TF_PASSERT\((.+),([^)]+)\)(.*)$/s){
     $out_line = $1 . $4 . "\n";
-    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_($2))) THEN\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_($2, pio_tf_comm_))) THEN\n";
     $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
     $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: Assertion failed :\",&\n";
@@ -396,9 +410,23 @@ sub transform_src
     $out_line = $out_line . $1 . "  RETURN\n";
     $out_line = $out_line . $1 . "END IF";
   }
+  elsif(/^(\s*)PIO_TF_CHECK_ERR\(([^,]+),([^,]+),(.+)\)(\s*)$/s){
+    $out_line = $1 . $5 . "\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_(($2) == PIO_NOERR, $3))) THEN\n";
+    $out_line = $out_line . $1 . "  call MPI_COMM_RANK($3, pio_tf_tmp_comm_rank_, pio_tf_retval_utest_)\n";
+    $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
+    $out_line = $out_line . $1 . "  IF (pio_tf_tmp_comm_rank_ == 0) THEN\n";
+    $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: PIO Function failed:\",&\n";
+    $out_line = $out_line . $1 . "      " . $4 . ",&\n";
+    $out_line = $out_line . $1 . "      \":\", __FILE__, \":\", __LINE__,&\n";
+    $out_line = $out_line . $1 . "      " . "\"($template_fname:$template_line_no)\"\n";
+    $out_line = $out_line . $1 . "  END IF\n";
+    $out_line = $out_line . $1 . "  RETURN\n";
+    $out_line = $out_line . $1 . "END IF";
+  }
   elsif(/^(\s*)PIO_TF_CHECK_ERR\(([^,]+),(.+)\)(\s*)$/s){
     $out_line = $1 . $4 . "\n";
-    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_(($2) == PIO_NOERR))) THEN\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_(($2) == PIO_NOERR, pio_tf_comm_))) THEN\n";
     $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
     $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: PIO Function failed:\",&\n";
@@ -455,8 +483,8 @@ sub transform_src
 # Returns 1 => if a generic template func is found, 0 otherwise
 sub find_gen_templ_funcs
 {
-  my($_, $ref_templ_typenames) = @_;
-  if(/^\s*PIO_TF_TEMPLATE\s*<(.*)>.*$/s){
+  my($str, $ref_templ_typenames) = @_;
+  if($str =~ /^\s*PIO_TF_TEMPLATE\s*<(.*)>.*$/s){
     if($1 eq ''){
       print "Error parsing template code, Check syntax of PIO_TF_TEMPLATE...\n";
       return 0;
@@ -555,7 +583,7 @@ sub update_auto_func_list_with_gen_templ
   my ($gen_templ_func_name);
   my (@modif_gen_templ_func_names);
   foreach $gen_templ_func_name (@{$ref_gen_templ_funcs_list}){
-    # Get the general template function names 
+    # Get the general template function names
     &generate_gen_templates($gen_templ_func_name, \@modif_gen_templ_func_names,
         $base_file_name, $ifline_num);
     my ($func_name);
@@ -573,20 +601,29 @@ sub get_default_test_main
   $out_line = $out_line . "  PROGRAM PIO_TF_Test_main_\n";
   $out_line = $out_line . "    USE pio_tutil\n";
   $out_line = $out_line . "    IMPLICIT NONE\n";
-  $out_line = $out_line . "    INTEGER ierr\n";
+  $out_line = $out_line . "    INTEGER, PARAMETER :: NREARRS = 2\n";
+  $out_line = $out_line . "    INTEGER :: rearrs(NREARRS) = (/pio_rearr_subset,pio_rearr_box/)\n";
+  $out_line = $out_line . "    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: rearrs_info(NREARRS) = (/\"PIO_REARR_SUBSET\",\"PIO_REARR_BOX   \"/)\n";
+  $out_line = $out_line . "    INTEGER i, ierr\n";
   $out_line = $out_line . "\n";
   $out_line = $out_line . "    pio_tf_nerrs_total_=0\n";
   $out_line = $out_line . "    pio_tf_retval_utest_=0\n";
   $out_line = $out_line . "    CALL MPI_Init(ierr)\n";
-  $out_line = $out_line . "    CALL PIO_TF_Init_()\n";
-  $out_line = $out_line . "    CALL PIO_TF_Test_driver_()\n";
-  $out_line = $out_line . "    CALL PIO_TF_Finalize_()\n";
+  $out_line = $out_line . "    DO i=1,SIZE(rearrs)\n";
+  $out_line = $out_line . "      CALL PIO_TF_Init_(rearrs(i))\n";
+  $out_line = $out_line . "      IF (pio_tf_world_rank_ == 0) THEN\n";
+  $out_line = $out_line . "        WRITE(*,*) \"PIO_TF: Testing : \", trim(rearrs_info(i))\n";
+  $out_line = $out_line . "      END IF\n";
+  $out_line = $out_line . "      CALL PIO_TF_Test_driver_()\n";
+  $out_line = $out_line . "      CALL PIO_TF_Finalize_()\n";
+  $out_line = $out_line . "    END DO\n";
   $out_line = $out_line . "    IF (pio_tf_world_rank_ == 0) THEN\n";
   $out_line = $out_line . "      IF (pio_tf_nerrs_total_ == 0) THEN\n";
   $out_line = $out_line . "        IF (pio_tf_retval_utest_ == 0) THEN\n";
   $out_line = $out_line . "          WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF: \",&\n";
   $out_line = $out_line . "           \"All tests\", \"---------\", \"PASSED\"\n";
   $out_line = $out_line . "        ELSE\n";
+  $out_line = $out_line . "          pio_tf_nerrs_total_ = pio_tf_nerrs_total_ + 1\n";
   $out_line = $out_line . "          WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF: \",&\n";
   $out_line = $out_line . "           \"Test driver\", \"---------\", \"FAILED\"\n";
   $out_line = $out_line . "        END IF\n";
@@ -630,10 +667,10 @@ sub get_default_test_driver
     $out_line = $out_line . "    END IF\n";
     $out_line = $out_line . "    IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . "      IF (pio_tf_retval_utest_ == 0) THEN\n";
-    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . "          \"$_\",\"-----------\", \"PASSED\"\n";
     $out_line = $out_line . "      ELSE\n";
-    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . "          \"$_\",\"-----------\", \"FAILED\"\n";
     $out_line = $out_line . "      END IF\n";
     $out_line = $out_line . "    END IF\n";
@@ -756,7 +793,7 @@ sub process_template_file
   $orig_line = "";
   while(){
     my ($out_line);
-  
+
     $orig_line = $orig_line . $_;
     chomp;
     $_ = $in_line . $_;
@@ -769,7 +806,7 @@ sub process_template_file
       # Strip off comments at the end of the line
       if(/^([^!]*)!(.*)$/s){
         if($verbose){ print "Stripping comment: $2\n"; }
-        $_ = $1; 
+        $_ = $1;
       }
       if(/^\s*PROGRAM.*/si){
         # Report error and exit if the template has a PROGRAM keyword
@@ -829,8 +866,8 @@ ()
 # Read input args
 GetOptions(
   # Annotate generated source with template line numbers etc
-  "annotate-source"		=>	\$annotate_source,
-  "out=s"				      =>	\$output_fname,
+  "annotate-source"             =>      \$annotate_source,
+  "out=s"                                     =>        \$output_fname,
   "verbose"           =>  \$verbose
 );
 
@@ -853,4 +890,3 @@ ()
 
 if($verbose){ print "Reading input args complete\n" }
 &process_template_file($template_fname, $output_fname);
-
diff --git a/externals/pio2/tests/general/util/pio_tutil.F90 b/src/externals/pio2/tests/general/util/pio_tutil.F90
similarity index 72%
rename from externals/pio2/tests/general/util/pio_tutil.F90
rename to src/externals/pio2/tests/general/util/pio_tutil.F90
index 398d9d2fbe5..43c0b634b4a 100644
--- a/externals/pio2/tests/general/util/pio_tutil.F90
+++ b/src/externals/pio2/tests/general/util/pio_tutil.F90
@@ -13,9 +13,10 @@ MODULE pio_tutil
     ENUMERATOR  ::  IARG_STRIDE_SIDX = 1
     ENUMERATOR  ::  IARG_NUM_IO_TASKS_SIDX
     ENUMERATOR  ::  IARG_NUM_AGGREGATORS_SIDX
+    ENUMERATOR  ::  IARG_LOG_LEVEL_SIDX
     ! Unfortunately since fortran starts with index 1 we need the
     ! hack below. Don't forget to update when adding more argvs
-    ENUMERATOR  ::  NUM_IARGS = IARG_NUM_AGGREGATORS_SIDX
+    ENUMERATOR  ::  NUM_IARGS = IARG_LOG_LEVEL_SIDX
     ENUMERATOR  ::  IARG_MAX_SIDX = NUM_IARGS
   END ENUM
 
@@ -25,6 +26,7 @@ MODULE pio_tutil
 
   ! MPI info
   INTEGER ::  pio_tf_world_rank_, pio_tf_world_sz_
+  INTEGER :: pio_tf_tmp_comm_rank_, pio_tf_tmp_comm_sz_
   INTEGER :: pio_tf_comm_
 
   ! REAL types
@@ -39,11 +41,11 @@ MODULE pio_tutil
   ! PIO_TF_TEST_RES_FMT is used for formatted test result output
   ! -- Useful for writes like
   ! HEADER_STRING, TEST_DESC, FOOTER_STRING, TEST_STATUS
-  ! "PIO_TF: Test no: 12", "Test name, desc etc", "-----", "PASSED" 
+  ! "PIO_TF: Test no: 12", "Test name, desc etc", "-----", "PASSED"
   CHARACTER(LEN=*), PARAMETER :: PIO_TF_TEST_RES_FMT = "(A20,T22,A40,T64,A6,T72,A6)"
   ! -- Useful for writes like
   ! HEADER_STRING, NUMBER_OF_TESTS, FOOTER_STRING, TEST_STATUS
-  ! "PIO_TF: [", 3, "] -----", "FAILED" 
+  ! "PIO_TF: [", 3, "] -----", "FAILED"
   CHARACTER(LEN=*), PARAMETER :: PIO_TF_TEST_RES_FMT2 = "(A20,T22,I5,T28,A10,T62,A16)"
   CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: pio_tf_tmp_log_str_
 
@@ -64,13 +66,23 @@ MODULE pio_tutil
   PUBLIC  :: PIO_TF_Get_data_types
   PUBLIC  :: PIO_TF_Check_val_
   ! Private functions
+  PRIVATE :: PIO_TF_Check_int_arr_arr_
   PRIVATE :: PIO_TF_Check_int_arr_val, PIO_TF_Check_int_arr_arr
   PRIVATE :: PIO_TF_Check_int_arr_arr_tol
+  PRIVATE :: PIO_TF_Check_2d_int_arr_arr
+  PRIVATE :: PIO_TF_Check_3d_int_arr_arr
   PRIVATE :: PIO_TF_Check_real_arr_val, PIO_TF_Check_real_arr_arr
+  PRIVATE :: PIO_TF_Check_real_arr_arr_tol_
   PRIVATE :: PIO_TF_Check_real_arr_arr_tol
+  PRIVATE :: PIO_TF_Check_2d_real_arr_arr
+  PRIVATE :: PIO_TF_Check_3d_real_arr_arr
   PRIVATE :: PIO_TF_Check_double_arr_val, PIO_TF_Check_double_arr_arr
+  PRIVATE :: PIO_TF_Check_double_arr_arr_tol_
   PRIVATE :: PIO_TF_Check_double_arr_arr_tol
+  PRIVATE :: PIO_TF_Check_2d_double_arr_arr
+  PRIVATE :: PIO_TF_Check_3d_double_arr_arr
   PRIVATE :: PIO_TF_Check_char_str_str
+  PRIVATE :: PIO_TF_Get_idx_from_1d_idx
 
   ! Note that the tolerance value provided is ignored when comparing two
   ! integer arrays
@@ -79,18 +91,24 @@ MODULE pio_tutil
         PIO_TF_Check_int_arr_val,     &
         PIO_TF_Check_int_arr_arr,     &
         PIO_TF_Check_int_arr_arr_tol, &
+        PIO_TF_Check_2d_int_arr_arr,  &
+        PIO_TF_Check_3d_int_arr_arr,  &
         PIO_TF_Check_real_arr_val,    &
         PIO_TF_Check_real_arr_arr,    &
+        PIO_TF_Check_2d_real_arr_arr, &
+        PIO_TF_Check_3d_real_arr_arr, &
         PIO_TF_Check_real_arr_arr_tol,&
         PIO_TF_Check_double_arr_val,  &
         PIO_TF_Check_double_arr_arr,  &
+        PIO_TF_Check_2d_double_arr_arr,&
+        PIO_TF_Check_3d_double_arr_arr,&
         PIO_TF_Check_double_arr_arr_tol,&
         PIO_TF_Check_char_str_str
   END INTERFACE
 
 CONTAINS
   ! Initialize Testing framework - Internal (Not directly used by unit tests)
-  SUBROUTINE  PIO_TF_Init_
+  SUBROUTINE  PIO_TF_Init_(rearr)
 #ifdef TIMING
    use perf_mod
 #endif
@@ -99,6 +117,7 @@ SUBROUTINE  PIO_TF_Init_
 #else
     include 'mpif.h'
 #endif
+    INTEGER, INTENT(IN) :: rearr
     INTEGER ierr
 
     CALL MPI_COMM_DUP(MPI_COMM_WORLD, pio_tf_comm_, ierr);
@@ -142,16 +161,25 @@ SUBROUTINE  PIO_TF_Init_
           pio_tf_num_io_tasks_,       &
           pio_tf_num_aggregators_,    &
           pio_tf_stride_,             &
-          PIO_rearr_subset,              &
+          rearr,                      &
           pio_tf_iosystem_,           &
           base=0)
 
     ! Set PIO to bcast error
     CALL PIO_seterrorhandling(pio_tf_iosystem_, PIO_BCAST_ERROR)
+
+    ! Set PIO logging level
+    ierr = PIO_set_log_level(pio_tf_log_level_)
+    if(ierr /= PIO_NOERR) then
+      PRINT *, "PIO_TF: Error setting PIO logging level"
+    end if
   END SUBROUTINE PIO_TF_Init_
 
   ! Finalize Testing framework - Internal (Not directly used by unit tests)
   SUBROUTINE  PIO_TF_Finalize_
+#ifdef TIMING
+   use perf_mod
+#endif
 #ifndef NO_MPIMOD
     use mpi
 #else
@@ -166,36 +194,45 @@ SUBROUTINE  PIO_TF_Finalize_
 
     ! Finalize PIO
     CALL PIO_finalize(pio_tf_iosystem_, ierr);
+    CALL MPI_COMM_FREE(pio_tf_comm_, ierr);
+
+#ifdef TIMING
+    call t_finalizef()
+#endif
   END SUBROUTINE PIO_TF_Finalize_
 
   ! Collective assert - Internal (Not directly used by unit tests)
   ! Each processes passes in its local assert condition and the function
   ! returns the global assert condition
-  LOGICAL FUNCTION PIO_TF_Passert_(local_result)
+  LOGICAL FUNCTION PIO_TF_Passert_(local_result, comm)
 #ifndef NO_MPIMOD
     use mpi
 #else
     include 'mpif.h'
 #endif
     LOGICAL, INTENT(IN) ::  local_result
+    INTEGER, INTENT(IN) ::  comm
     LOGICAL :: global_result
     LOGICAL :: failed, all_failed
     INTEGER :: rank, ierr
     LOGICAL, DIMENSION(:), ALLOCATABLE :: failed_ranks
 
-    CALL MPI_ALLREDUCE(local_result, global_result, 1, MPI_LOGICAL, MPI_LAND, pio_tf_comm_, ierr)
+    CALL MPI_COMM_RANK(comm, pio_tf_tmp_comm_rank_, ierr)
+    CALL MPI_COMM_SIZE(comm, pio_tf_tmp_comm_sz_, ierr)
+
+    CALL MPI_ALLREDUCE(local_result, global_result, 1, MPI_LOGICAL, MPI_LAND, comm, ierr)
     IF (.NOT. global_result) THEN
       failed = .NOT. local_result
 !      IF (pio_tf_world_rank_ == 0) THEN
-        ALLOCATE(failed_ranks(pio_tf_world_sz_))
+        ALLOCATE(failed_ranks(pio_tf_tmp_comm_sz_))
 !      END IF
       ! Gather the ranks where assertion failed
-      CALL MPI_GATHER(failed, 1, MPI_LOGICAL, failed_ranks, 1, MPI_LOGICAL, 0, pio_tf_comm_, ierr)
+      CALL MPI_GATHER(failed, 1, MPI_LOGICAL, failed_ranks, 1, MPI_LOGICAL, 0, comm, ierr)
 
       ! Display the ranks where the assertion failed
-      IF (pio_tf_world_rank_ == 0) THEN
+      IF (pio_tf_tmp_comm_rank_ == 0) THEN
         all_failed = .TRUE.
-        DO rank=1,pio_tf_world_sz_
+        DO rank=1,pio_tf_tmp_comm_sz_
           IF (.NOT. failed_ranks(rank)) THEN
             all_failed = .FALSE.
             ! Thank you - f90
@@ -207,7 +244,7 @@ LOGICAL FUNCTION PIO_TF_Passert_(local_result)
         ELSE
           PRINT *, "PIO_TF: Fatal Error: Assertion failed on following processes: "
           WRITE(*,"(A8)",ADVANCE="NO") "PIO_TF: "
-          DO rank=1,pio_tf_world_sz_
+          DO rank=1,pio_tf_tmp_comm_sz_
             IF (failed_ranks(rank)) THEN
               WRITE(*,"(I5,A)",ADVANCE="NO") rank-1, ","
             END IF
@@ -487,7 +524,52 @@ SUBROUTINE PIO_TF_Get_data_types(data_types, data_type_descs, num_data_types)
 
   END SUBROUTINE PIO_TF_Get_data_types
 
-  LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
+  ! Get original (multi-d) index string from 1d (reshaped) index
+  SUBROUTINE PIO_TF_Get_idx_from_1d_idx(idx_1d, arr_shape, idx_str)
+    INTEGER, INTENT(IN) :: idx_1d
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
+    CHARACTER(LEN=*), INTENT(OUT) :: idx_str
+
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: fmt_str
+    ! Number of elems in that dim in "1D view"
+    INTEGER, DIMENSION(:), ALLOCATABLE :: dim_wgt
+    INTEGER, DIMENSION(:), ALLOCATABLE :: idx_md
+    INTEGER :: tmp_idx
+    INTEGER :: i, sz
+
+    idx_str = ""
+
+    sz = SIZE(arr_shape)
+    ALLOCATE(dim_wgt(sz))
+    ALLOCATE(idx_md(sz))
+
+    ! Assign place weights = num of elems in that dim in "1D view"
+    dim_wgt = 1
+    DO i=2,sz
+      dim_wgt(i) = dim_wgt(i-1) * arr_shape(i-1)
+    END DO
+
+    ! Convert 1d reshaped index to original multi-d index
+    tmp_idx = idx_1d - 1
+    idx_md = 0
+    DO i=sz,1,-1
+      idx_md(i) = tmp_idx / dim_wgt(i) + 1
+      tmp_idx = MOD(tmp_idx, dim_wgt(i))
+    END DO
+
+    IF(sz == 1) THEN
+      WRITE(fmt_str, *) "(",sz,"(I5)", ")"
+    ELSE
+      WRITE(fmt_str, *) "(",sz,"(I5,',')", ")"
+    END IF
+    WRITE(idx_str,fmt_str) idx_md
+
+    DEALLOCATE(idx_md)
+    DEALLOCATE(dim_wgt)
+
+  END SUBROUTINE PIO_TF_Get_idx_from_1d_idx
+
+  LOGICAL FUNCTION PIO_TF_Check_int_arr_arr_(arr, exp_arr, arr_shape)
 #ifndef NO_MPIMOD
     USE mpi
 #else
@@ -495,6 +577,8 @@ LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
 #endif
     INTEGER, DIMENSION(:), INTENT(IN) :: arr
     INTEGER, DIMENSION(:), INTENT(IN) :: exp_arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: idx_str
     INTEGER :: arr_sz, i, ierr
     ! Not equal at id = nequal_idx
     INTEGER :: nequal_idx
@@ -536,14 +620,23 @@ LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
       IF (pio_tf_world_rank_ == 0) THEN
          DO i=1,pio_tf_world_sz_
             IF(gfail_info(i) % idx /= -1) THEN
-               PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", gfail_info(i) % idx, "]=", &
+               CALL PIO_TF_Get_idx_from_1d_idx(gfail_info(i) % idx, arr_shape, idx_str)
+               PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[",&
+                    trim(idx_str), "]=",&
                     gfail_info(i) % val, ", Expected = ", gfail_info(i) % exp_val
             END IF
          END DO
       END IF
       deallocate(gfail_info)
    end if
-    PIO_TF_Check_int_arr_arr = gequal
+    PIO_TF_Check_int_arr_arr_ = gequal
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: exp_arr
+
+    PIO_TF_Check_int_arr_arr = PIO_TF_Check_int_arr_arr_(arr, exp_arr, SHAPE(arr))
   END FUNCTION
 
   ! Note that the tolerance value is ignored when comparing two integer arrays
@@ -568,7 +661,45 @@ LOGICAL FUNCTION PIO_TF_Check_int_arr_val(arr, val)
     DEALLOCATE(arr_val)
   END FUNCTION
 
-  LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
+  LOGICAL FUNCTION PIO_TF_Check_2d_int_arr_arr(arr, exp_arr)
+    INTEGER, DIMENSION(:,:), INTENT(IN) :: arr
+    INTEGER, DIMENSION(:,:), INTENT(IN) :: exp_arr
+
+    INTEGER, DIMENSION(:), ALLOCATABLE :: arr_val
+    INTEGER, DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_2d_int_arr_arr = PIO_TF_Check_int_arr_arr_(arr_val, exp_arr_val,&
+                                    SHAPE(arr))
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_3d_int_arr_arr(arr, exp_arr)
+    INTEGER, DIMENSION(:,:,:), INTENT(IN) :: arr
+    INTEGER, DIMENSION(:,:,:), INTENT(IN) :: exp_arr
+
+    INTEGER, DIMENSION(:), ALLOCATABLE :: arr_val
+    INTEGER, DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_3d_int_arr_arr = PIO_TF_Check_int_arr_arr_(arr_val, exp_arr_val,&
+                                    SHAPE(arr))
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol_(arr, exp_arr, arr_shape, tol)
 #ifndef NO_MPIMOD
     USE mpi
 #else
@@ -576,8 +707,10 @@ LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
 #endif
     REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: arr
     REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: exp_arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
     REAL, INTENT(IN) :: tol
 
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: idx_str
     INTEGER :: arr_sz, i, ierr
     ! Not equal at id = nequal_idx
     REAL(KIND=fc_real) :: nequal_idx
@@ -619,15 +752,25 @@ LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
       IF (pio_tf_world_rank_ == 0) THEN
         DO i=1,pio_tf_world_sz_
           IF(INT(gfail_info(i) % idx) /= -1) THEN
-            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", &
-                 INT(gfail_info(i) % idx), "]=", gfail_info(i) % val, ", Expected = ", gfail_info(i) % exp_val
+            CALL PIO_TF_Get_idx_from_1d_idx(INT(gfail_info(i)%idx), arr_shape, idx_str)
+            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", trim(idx_str),&
+                 "]=", gfail_info(i) % val, ", Expected = ", gfail_info(i) % exp_val
           END IF
         END DO
       END IF
       DEALLOCATE(gfail_info)
    END IF
 
-    PIO_TF_Check_real_arr_arr_tol = gequal
+    PIO_TF_Check_real_arr_arr_tol_ = gequal
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
+    REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: arr
+    REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: exp_arr
+    REAL, INTENT(IN) :: tol
+
+    PIO_TF_Check_real_arr_arr_tol = PIO_TF_Check_real_arr_arr_tol_(arr, exp_arr,&
+                                SHAPE(arr), 0.0)
   END FUNCTION
 
   LOGICAL FUNCTION PIO_TF_Check_real_arr_arr(arr, exp_arr)
@@ -648,7 +791,45 @@ LOGICAL FUNCTION PIO_TF_Check_real_arr_val(arr, val)
     DEALLOCATE(arr_val)
   END FUNCTION
 
-  LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
+  LOGICAL FUNCTION PIO_TF_Check_2d_real_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_real), DIMENSION(:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_real), DIMENSION(:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_2d_real_arr_arr = PIO_TF_Check_real_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_3d_real_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_real), DIMENSION(:,:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_real), DIMENSION(:,:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_3d_real_arr_arr = PIO_TF_Check_real_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol_(arr, exp_arr, arr_shape, tol)
 #ifndef NO_MPIMOD
     USE mpi
 #else
@@ -659,7 +840,10 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
 #endif
     REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: arr
     REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: exp_arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
     REAL, INTENT(IN) :: tol
+
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: idx_str
     INTEGER :: arr_sz, i, ierr
     ! Not equal at id = nequal_idx
     REAL(KIND=fc_double) :: nequal_idx
@@ -703,8 +887,9 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
       IF (pio_tf_world_rank_ == 0) THEN
         DO i=1,pio_tf_world_sz_
           IF(INT(gfail_info(i) % idx) /= -1) THEN
-            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", &
-                  INT(gfail_info(i) % idx), "]=", gfail_info(i) % val, &
+            CALL PIO_TF_Get_idx_from_1d_idx(INT(gfail_info(i)%idx), arr_shape, idx_str)
+            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[",&
+                  trim(idx_str), "]=", gfail_info(i) % val,&
                   ", Expected = ", gfail_info(i) % exp_val
           END IF
         END DO
@@ -712,7 +897,16 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
       DEALLOCATE(gfail_info)
    END IF
 
-    PIO_TF_Check_double_arr_arr_tol = gequal
+    PIO_TF_Check_double_arr_arr_tol_ = gequal
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
+    REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: arr
+    REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: exp_arr
+    REAL, INTENT(IN) :: tol
+
+    PIO_TF_Check_double_arr_arr_tol = PIO_TF_Check_double_arr_arr_tol_(arr, exp_arr,&
+                                    SHAPE(arr), 0.0)
   END FUNCTION
 
   LOGICAL FUNCTION PIO_TF_Check_double_arr_arr(arr, exp_arr)
@@ -733,11 +927,53 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_val(arr, val)
     DEALLOCATE(arr_val)
   END FUNCTION
 
+  LOGICAL FUNCTION PIO_TF_Check_2d_double_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_double), DIMENSION(:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_double), DIMENSION(:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_2d_double_arr_arr = PIO_TF_Check_double_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_3d_double_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_double), DIMENSION(:,:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_double), DIMENSION(:,:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_3d_double_arr_arr = PIO_TF_Check_double_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
   LOGICAL FUNCTION PIO_TF_Check_char_str_str(str1, str2)
     CHARACTER(LEN=*), INTENT(IN)  :: str1
     CHARACTER(LEN=*), INTENT(IN)  :: str2
 
-    PIO_TF_Check_char_str_str = .TRUE.
+    IF (str1 == str2) THEN
+      PIO_TF_Check_char_str_str = .TRUE.
+    ELSE
+      PIO_TF_Check_char_str_str = .FALSE.
+    END IF
   END FUNCTION
 
   ! Parse and process input arguments like "--pio-tf-stride=2" passed
@@ -761,6 +997,8 @@ SUBROUTINE Parse_and_process_input(argv)
         READ(argv(pos+1:), *) pio_tf_num_aggregators_
       ELSE IF (argv(:pos) == "--pio-tf-stride=") THEN
         READ(argv(pos+1:), *) pio_tf_stride_
+      ELSE IF (argv(:pos) == "--pio-tf-log-level=") THEN
+        READ(argv(pos+1:), *) pio_tf_log_level_
       ELSE IF (argv(:pos) == "--pio-tf-input-file=") THEN
         PRINT *, "This option is not implemented yet"
       END IF
@@ -791,12 +1029,14 @@ SUBROUTINE Read_input()
       send_buf(IARG_STRIDE_SIDX) = pio_tf_stride_
       send_buf(IARG_NUM_IO_TASKS_SIDX) = pio_tf_num_io_tasks_
       send_buf(IARG_NUM_AGGREGATORS_SIDX) = pio_tf_num_aggregators_
+      send_buf(IARG_LOG_LEVEL_SIDX) = pio_tf_log_level_
     END IF
     ! Make sure all processes get the input args
     CALL MPI_BCAST(send_buf, NUM_IARGS, MPI_INTEGER, 0, pio_tf_comm_, ierr)
     pio_tf_stride_ = send_buf(IARG_STRIDE_SIDX)
     pio_tf_num_io_tasks_ = send_buf(IARG_NUM_IO_TASKS_SIDX)
     pio_tf_num_aggregators_ = send_buf(IARG_NUM_AGGREGATORS_SIDX)
+    pio_tf_log_level_ = send_buf(IARG_LOG_LEVEL_SIDX)
 
   END SUBROUTINE Read_input
 END MODULE pio_tutil
diff --git a/externals/pio2/tests/performance/CMakeLists.txt b/src/externals/pio2/tests/performance/CMakeLists.txt
similarity index 56%
rename from externals/pio2/tests/performance/CMakeLists.txt
rename to src/externals/pio2/tests/performance/CMakeLists.txt
index ea9efbf0a51..fce71424308 100644
--- a/externals/pio2/tests/performance/CMakeLists.txt
+++ b/src/externals/pio2/tests/performance/CMakeLists.txt
@@ -3,22 +3,22 @@
 #==============================================================================
 
 add_executable (pioperf EXCLUDE_FROM_ALL
-                pioperformance.F90)
+  pioperformance.F90)
 target_link_libraries (pioperf piof)
 add_dependencies (tests pioperf)
 
 if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
-    target_compile_options (pioperf
-        PRIVATE -ffree-line-length-none)
+  target_compile_options (pioperf
+    PRIVATE -ffree-line-length-none)
 endif()
 
 if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
-   set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
-#    target_compile_options (gptl
-#        PRIVATE -mismatch_all)
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
 endif ()
 
-if (PIO_HDF5_LOGGING) 
-    target_compile_definitions (pioperf 
-        PUBLIC LOGGING)
+if (PIO_HDF5_LOGGING)
+  target_compile_definitions (pioperf
+    PUBLIC LOGGING)
 endif ()
diff --git a/externals/pio2/tests/performance/gensimple.pl b/src/externals/pio2/tests/performance/gensimple.pl
similarity index 100%
rename from externals/pio2/tests/performance/gensimple.pl
rename to src/externals/pio2/tests/performance/gensimple.pl
diff --git a/externals/pio2/tests/performance/kt.PIO1.perfmakefile b/src/externals/pio2/tests/performance/kt.PIO1.perfmakefile
similarity index 100%
rename from externals/pio2/tests/performance/kt.PIO1.perfmakefile
rename to src/externals/pio2/tests/performance/kt.PIO1.perfmakefile
diff --git a/externals/pio2/tests/performance/pioperformance.F90 b/src/externals/pio2/tests/performance/pioperformance.F90
similarity index 100%
rename from externals/pio2/tests/performance/pioperformance.F90
rename to src/externals/pio2/tests/performance/pioperformance.F90
diff --git a/src/externals/pio2/tests/unit/CMakeLists.txt b/src/externals/pio2/tests/unit/CMakeLists.txt
new file mode 100644
index 00000000000..eaf7c0f5694
--- /dev/null
+++ b/src/externals/pio2/tests/unit/CMakeLists.txt
@@ -0,0 +1,63 @@
+include (LibMPI)
+
+include_directories("${CMAKE_SOURCE_DIR}/tests/unit")
+
+#==============================================================================
+#  PREPARE FOR TESTING
+#==============================================================================
+
+file (COPY "./input.nl"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file (COPY "./not_netcdf.ieee"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+#==============================================================================
+#  DEFINE THE TARGETS AND TESTS
+#==============================================================================
+
+set (SRCS basic_tests.F90
+  driver.F90
+  global_vars.F90
+  ncdf_tests.F90)
+
+add_executable (pio_unit_test EXCLUDE_FROM_ALL ${SRCS})
+target_link_libraries (pio_unit_test piof)
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (pio_unit_test
+    PRIVATE -ffree-line-length-none)
+endif()
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
+endif ()
+
+add_dependencies (tests pio_unit_test)
+
+# Test Timeout in seconds.
+set (DEFAULT_TEST_TIMEOUT 60)
+
+# All tests need a certain number of tasks, but they should be able to
+# run successfully with more than they need. Test this by providing an
+# extra processor for each C test.
+set (AT_LEAST_TWO_TASKS 3)
+set (AT_LEAST_THREE_TASKS 4)
+set (AT_LEAST_FOUR_TASKS 5)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_unit_test
+    COMMAND pio_unit_test)
+  set_tests_properties(pio_unit_test
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_unit_test
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_unit_test
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+if (PIO_HDF5_LOGGING)
+  target_compile_definitions (pio_unit_test
+    PUBLIC LOGGING)
+endif ()
diff --git a/externals/pio2/tests/unit/Levy_Notes b/src/externals/pio2/tests/unit/Levy_Notes
similarity index 100%
rename from externals/pio2/tests/unit/Levy_Notes
rename to src/externals/pio2/tests/unit/Levy_Notes
diff --git a/externals/pio2/tests/unit/README b/src/externals/pio2/tests/unit/README
similarity index 100%
rename from externals/pio2/tests/unit/README
rename to src/externals/pio2/tests/unit/README
diff --git a/externals/pio2/tests/unit/basic_tests.F90 b/src/externals/pio2/tests/unit/basic_tests.F90
similarity index 87%
rename from externals/pio2/tests/unit/basic_tests.F90
rename to src/externals/pio2/tests/unit/basic_tests.F90
index de41cef69d4..daad01babca 100644
--- a/externals/pio2/tests/unit/basic_tests.F90
+++ b/src/externals/pio2/tests/unit/basic_tests.F90
@@ -33,7 +33,7 @@ Subroutine test_create(test_id, err_msg)
 
       ! Local Vars
       character(len=str_len) :: filename
-      integer                :: iotype, ret_val, pio_dim
+      integer                :: iotype, ret_val, ret_val2, pio_dim
 
       err_msg = "no_error"
 
@@ -51,7 +51,7 @@ Subroutine test_create(test_id, err_msg)
         ! Error in PIO_createfile
          print *,' ret_val = ', ret_val
         err_msg = "Could not create " // trim(filename)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
       
       call mpi_barrier(mpi_comm_world,ret_val)
@@ -62,7 +62,7 @@ Subroutine test_create(test_id, err_msg)
           ! Error in PIO_enddef
           err_msg = "Could not end define mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
       call PIO_closefile(pio_file)
@@ -73,7 +73,7 @@ Subroutine test_create(test_id, err_msg)
       if (ret_val .ne. PIO_NOERR) then
         ! Error in PIO_openfile
         err_msg = "Could not open " // trim(filename)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Close file
@@ -85,7 +85,7 @@ Subroutine test_create(test_id, err_msg)
         if (ret_val .ne. PIO_NOERR) then
           ! Error in PIO_createfile
           err_msg = "Could not clobber " // trim(filename)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Leave define mode
@@ -94,7 +94,7 @@ Subroutine test_create(test_id, err_msg)
           ! Error in PIO_enddef
           err_msg = "Could not end define mode in clobbered file"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Close file
@@ -113,7 +113,7 @@ Subroutine test_create(test_id, err_msg)
           err_msg = "Was able to clobber file despite PIO_NOCLOBBER"
           ret_val = PIO_enddef(pio_file)
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
 
@@ -137,7 +137,7 @@ Subroutine test_open(test_id, err_msg)
 
       ! Local Vars
       character(len=str_len) :: filename
-      integer                :: iotype, ret_val
+      integer                :: iotype, ret_val, ret_val2
 
       ! Data used to test writing
       integer,          dimension(3) :: data_buffer, compdof
@@ -147,6 +147,12 @@ Subroutine test_open(test_id, err_msg)
       integer :: unlimdimid
       type(var_desc_t)               :: pio_var
 
+      ! These will be used to set chunk cache sizes in netCDF-4/HDF5
+      ! files.
+      integer(kind=PIO_OFFSET_KIND) :: chunk_cache_size
+      integer(kind=PIO_OFFSET_KIND) :: chunk_cache_nelems
+      real :: chunk_cache_preemption
+
       err_msg = "no_error"
       dims(1) = 3*ntasks
       compdof = 3*my_rank+(/1,2,3/)  ! Where in the global array each task writes
@@ -166,7 +172,7 @@ Subroutine test_open(test_id, err_msg)
         ! Error in PIO_openfile
         err_msg = "Successfully opened file that doesn't exist"
         call PIO_closefile(pio_file)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Open existing file, write data to it (for binary file, need to create new file)
@@ -178,17 +184,16 @@ Subroutine test_open(test_id, err_msg)
       if (ret_val .ne. PIO_NOERR) then
         ! Error in PIO_openfile (or PIO_createfile)
         err_msg = "Could not open " // trim(filename) // " in write mode"
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Enter define mode for netcdf files
       if (is_netcdf(iotype)) then
         ret_val = PIO_redef(pio_file)
         if (ret_val .ne. PIO_NOERR) then
-          ! Error in PIO_redef
           err_msg = "Could not enter redef mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Define a new dimension N
@@ -197,7 +202,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_def_dim
           err_msg = "Could not define dimension N"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Define a new variable foo
@@ -207,7 +212,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_def_var
           err_msg = "Could not define variable foo"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Leave define mode
@@ -217,7 +222,7 @@ Subroutine test_open(test_id, err_msg)
            print *,__FILE__,__LINE__,ret_val
           err_msg = "Could not end define mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
 
@@ -229,7 +234,7 @@ Subroutine test_open(test_id, err_msg)
         ! Error in PIO_write_darray
         err_msg = "Could not write data"
         call PIO_closefile(pio_file)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Close file
@@ -242,7 +247,7 @@ Subroutine test_open(test_id, err_msg)
         if (ret_val .ne. PIO_NOERR) then
           ! Error opening file
           err_msg = "Could not open file in NoWrite mode"
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Try to write (should fail)
@@ -254,7 +259,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_write_darray
           err_msg = "Wrote to file opened in NoWrite mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         call mpi_barrier(MPI_COMM_WORLD,ret_val)
@@ -267,25 +272,25 @@ Subroutine test_open(test_id, err_msg)
           err_msg = "Error in read_darray"
           call PIO_closefile(pio_file)
           print *,__FILE__,__LINE__,err_msg
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
         if(any(data_buffer /= my_rank)) then
           err_msg = "Error reading data"
           call PIO_closefile(pio_file)
           print *,__FILE__,__LINE__,iotype, trim(err_msg), data_buffer
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
+        ret_val = PIO_set_log_level(3)
         ret_val = PIO_inq_unlimdim(pio_file, unlimdimid)
         if(unlimdimid /= -1) then
            err_msg = "Error in inq_unlimdim"
            call PIO_closefile(pio_file)
            print *,__FILE__,__LINE__,iotype, trim(err_msg)
-           call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+           call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
-
+        ret_val = PIO_set_log_level(0)
         
-
         ! Close file
         call PIO_closefile(pio_file)
       end if
@@ -303,7 +308,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_openfile
           err_msg = "Opened a non-netcdf file as netcdf"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
 
diff --git a/externals/pio2/tests/unit/driver.F90 b/src/externals/pio2/tests/unit/driver.F90
similarity index 90%
rename from externals/pio2/tests/unit/driver.F90
rename to src/externals/pio2/tests/unit/driver.F90
index 7dbcfada7ed..a955339ee08 100644
--- a/externals/pio2/tests/unit/driver.F90
+++ b/src/externals/pio2/tests/unit/driver.F90
@@ -23,9 +23,10 @@ Program pio_unit_test_driver
        ltest_netcdf4c,     &
        ltest_pnetcdf,    &
        stride
-#if defined( _NETCDF4) && defined(LOGGING)
-  integer, external :: nc_set_log_level2
-#endif
+  integer ret_val
+  character(len=pio_max_name) :: errmsg
+  character(len=pio_max_name) :: expected
+  
   ! Set up MPI
   call MPI_Init(ierr)
   call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
@@ -62,7 +63,7 @@ Program pio_unit_test_driver
 
      ! Ignore namelist values if PIO not built with correct options
      ! (i.e. don't test pnetcdf if not built with pnetcdf)
-
+     ret_val = PIO_set_log_level(2)
 #ifndef _NETCDF
      if (ltest_netcdf) then
         write(*,"(A,1x,A)") "WARNING: can not test netcdf files because PIO", &
@@ -116,11 +117,22 @@ Program pio_unit_test_driver
        PIO_rearr_subset,  & ! rearr
        pio_iosystem, base=1)     ! iosystem
 
-  call PIO_seterrorhandling(pio_iosystem, PIO_BCAST_ERROR)
+  call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR)
+  call PIO_seterrorhandling(PIO_DEFAULT, PIO_RETURN_ERROR)
 
   fail_cnt = 0
   test_cnt = 0
 
+  ! Test pio_strerror.
+  ret_val = PIO_strerror(-33, errmsg);
+  print *, 'errcode =', -33, ' strerror = ', errmsg
+  expected = 'NetCDF: Not a valid ID'
+  if (trim(errmsg) .ne. expected) then
+     err_msg = 'expected ' // trim(expected) // ' and got ' // trim(errmsg)
+     print *, err_msg
+     call parse(err_msg, fail_cnt)
+  end if
+     
   do test_id=1,ntest
      if (ltest(test_id)) then
         ! Make sure i is a valid test number
@@ -142,9 +154,7 @@ Program pio_unit_test_driver
                 write(*,"(A,I0)") "Error, not configured for test #", test_id
            call MPI_Abort(MPI_COMM_WORLD, 0, ierr)
         end select
-#if defined( _NETCDF4) && defined(LOGGING)
-        if(master_task) ierr = nc_set_log_level2(3)
-#endif
+        
         ! test_create()
         if (master_task) write(*,"(3x,A,1x)") "testing PIO_createfile..."
         call test_create(test_id, err_msg)
@@ -193,6 +203,9 @@ Program pio_unit_test_driver
   end if
 
   call PIO_finalize(pio_iosystem, ierr)
+#ifdef TIMING
+  call t_finalizef()
+#endif
   call MPI_Finalize(ierr)
   if(fail_cnt>0) then
      stop 1
diff --git a/externals/pio2/tests/unit/global_vars.F90 b/src/externals/pio2/tests/unit/global_vars.F90
similarity index 96%
rename from externals/pio2/tests/unit/global_vars.F90
rename to src/externals/pio2/tests/unit/global_vars.F90
index 9e8fced0353..f3347ba38c9 100644
--- a/externals/pio2/tests/unit/global_vars.F90
+++ b/src/externals/pio2/tests/unit/global_vars.F90
@@ -12,7 +12,7 @@ module global_vars
 
   include 'mpif.h' ! _EXTERNAL
 
-  integer, parameter :: str_len = 255, ntest=4
+  integer, parameter :: str_len = pio_max_name, ntest=4
    integer, parameter ::NETCDF =1, &
                         NETCDF4P=2, &
                         NETCDF4C=3, &
diff --git a/externals/pio2/tests/unit/input.nl b/src/externals/pio2/tests/unit/input.nl
similarity index 100%
rename from externals/pio2/tests/unit/input.nl
rename to src/externals/pio2/tests/unit/input.nl
diff --git a/externals/pio2/tests/unit/nc_set_log_level2.c b/src/externals/pio2/tests/unit/nc_set_log_level2.c
similarity index 64%
rename from externals/pio2/tests/unit/nc_set_log_level2.c
rename to src/externals/pio2/tests/unit/nc_set_log_level2.c
index deaf9087561..ad64d8e3309 100644
--- a/externals/pio2/tests/unit/nc_set_log_level2.c
+++ b/src/externals/pio2/tests/unit/nc_set_log_level2.c
@@ -3,8 +3,8 @@
 
 int nc_set_log_level2_(int *il)
 {
-  int i;
-  i = nc_set_log_level( *il );
-  return(i);
+    int i;
+    i = nc_set_log_level( *il );
+    return(i);
 }
 #endif
diff --git a/externals/pio2/tests/unit/ncdf_tests.F90 b/src/externals/pio2/tests/unit/ncdf_tests.F90
similarity index 99%
rename from externals/pio2/tests/unit/ncdf_tests.F90
rename to src/externals/pio2/tests/unit/ncdf_tests.F90
index 55459012de0..c811a6ddd0c 100644
--- a/externals/pio2/tests/unit/ncdf_tests.F90
+++ b/src/externals/pio2/tests/unit/ncdf_tests.F90
@@ -349,13 +349,14 @@ Subroutine test_nc4(test_id, err_msg)
        if (ret_val .ne. PIO_NOERR) then
           err_msg = "Could not set chunk cache"
           call PIO_closefile(pio_file)
+          return
        endif
        if (chunk_cache_size_in .ne. chunk_cache_size .or. chunk_cache_nelems_in .ne. chunk_cache_nelems .or. &
             chunk_cache_preemption_in .ne. chunk_cache_preemption) then
           err_msg = "Incorrect chunk cache values!"
           call PIO_closefile(pio_file)
+          return
        endif
-       return
     else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
        err_msg = "Did not get expected error when trying to set chunk cache for pnetcdf file"
        call PIO_closefile(pio_file)
@@ -441,14 +442,15 @@ Subroutine test_nc4(test_id, err_msg)
        if (ret_val .ne. PIO_NOERR) then
           err_msg = "Could not set variable chunk cache"
           call PIO_closefile(pio_file)
+          return
        endif
        if (chunk_cache_size_in .ne. chunk_cache_size .or. &
             chunk_cache_nelems_in .ne. chunk_cache_nelems .or. &
             chunk_cache_preemption_in .ne. chunk_cache_preemption) then
           err_msg = "Incorrect variable chunk cache values!"
           call PIO_closefile(pio_file)
+          return
        endif
-       return
     else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
        err_msg = "Did not get expected error when trying to get variable chunk cache for pnetcdf file"
        call PIO_closefile(pio_file)
diff --git a/externals/pio2/tests/unit/not_netcdf.ieee b/src/externals/pio2/tests/unit/not_netcdf.ieee
similarity index 100%
rename from externals/pio2/tests/unit/not_netcdf.ieee
rename to src/externals/pio2/tests/unit/not_netcdf.ieee
diff --git a/externals/pio2/tests/unit/test_names.c b/src/externals/pio2/tests/unit/test_names.c
similarity index 100%
rename from externals/pio2/tests/unit/test_names.c
rename to src/externals/pio2/tests/unit/test_names.c
diff --git a/externals/pio2/tests/unit/test_nc4.c b/src/externals/pio2/tests/unit/test_nc4.c
similarity index 100%
rename from externals/pio2/tests/unit/test_nc4.c
rename to src/externals/pio2/tests/unit/test_nc4.c
diff --git a/share/csm_share/README b/src/share/README
similarity index 79%
rename from share/csm_share/README
rename to src/share/README
index 1315e06147a..dff6073505b 100644
--- a/share/csm_share/README
+++ b/src/share/README
@@ -5,10 +5,10 @@
 
                   A description of csm_share
 
-This module exists to collect code shared between various CCSM components.
-Excluding this "shared code" module, CCSM components are built using disjoint
+This module exists to collect code shared between various CIME components.
+Excluding this "shared code" module, CIME components are built using disjoint
 sets of source code.  The use of this shared code is similar to the use of
-object code libraries where each subdirectory of csm_share is equivalant to
+object code libraries where each subdirectory of share is equivalant to
 one library.  While object library routines are accessed by linking to libraries
 during the load phase, these shared code routines are accessed by including the
 appropriate source code directory path during the compile phase.
@@ -25,7 +25,7 @@ Motivation for this code sharing includes:
 
 Current subsets ("libraries") of shared code only include:
 
-shr - very generic, general-purpose code that is likely to be useful to all
-      CCSM components.  CCSM components may be explicitly required to use some
+util - very generic, general-purpose code that is likely to be useful to all
+      CIME components.  CIME components may be explicitly required to use some
       parts of this code, for example the physical constants module.
 
diff --git a/share/shr_RandNum/include/dSFMT-common.h b/src/share/RandNum/include/dSFMT-common.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT-common.h
rename to src/share/RandNum/include/dSFMT-common.h
diff --git a/share/shr_RandNum/include/dSFMT-params.h b/src/share/RandNum/include/dSFMT-params.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT-params.h
rename to src/share/RandNum/include/dSFMT-params.h
diff --git a/share/shr_RandNum/include/dSFMT-params19937.h b/src/share/RandNum/include/dSFMT-params19937.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT-params19937.h
rename to src/share/RandNum/include/dSFMT-params19937.h
diff --git a/share/shr_RandNum/include/dSFMT.h b/src/share/RandNum/include/dSFMT.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT.h
rename to src/share/RandNum/include/dSFMT.h
diff --git a/share/shr_RandNum/src/dsfmt_f03/dSFMT.c b/src/share/RandNum/src/dsfmt_f03/dSFMT.c
similarity index 100%
rename from share/shr_RandNum/src/dsfmt_f03/dSFMT.c
rename to src/share/RandNum/src/dsfmt_f03/dSFMT.c
diff --git a/share/shr_RandNum/src/dsfmt_f03/dSFMT_interface.F90 b/src/share/RandNum/src/dsfmt_f03/dSFMT_interface.F90
similarity index 100%
rename from share/shr_RandNum/src/dsfmt_f03/dSFMT_interface.F90
rename to src/share/RandNum/src/dsfmt_f03/dSFMT_interface.F90
diff --git a/share/shr_RandNum/src/dsfmt_f03/dSFMT_utils.c b/src/share/RandNum/src/dsfmt_f03/dSFMT_utils.c
similarity index 100%
rename from share/shr_RandNum/src/dsfmt_f03/dSFMT_utils.c
rename to src/share/RandNum/src/dsfmt_f03/dSFMT_utils.c
diff --git a/share/shr_RandNum/src/kissvec/kissvec.c b/src/share/RandNum/src/kissvec/kissvec.c
similarity index 100%
rename from share/shr_RandNum/src/kissvec/kissvec.c
rename to src/share/RandNum/src/kissvec/kissvec.c
diff --git a/share/shr_RandNum/src/kissvec/kissvec_mod.F90 b/src/share/RandNum/src/kissvec/kissvec_mod.F90
similarity index 100%
rename from share/shr_RandNum/src/kissvec/kissvec_mod.F90
rename to src/share/RandNum/src/kissvec/kissvec_mod.F90
diff --git a/share/shr_RandNum/src/mt19937/mersennetwister_mod.F90 b/src/share/RandNum/src/mt19937/mersennetwister_mod.F90
similarity index 100%
rename from share/shr_RandNum/src/mt19937/mersennetwister_mod.F90
rename to src/share/RandNum/src/mt19937/mersennetwister_mod.F90
diff --git a/share/shr_RandNum/src/shr_RandNum_mod.F90 b/src/share/RandNum/src/shr_RandNum_mod.F90
similarity index 100%
rename from share/shr_RandNum/src/shr_RandNum_mod.F90
rename to src/share/RandNum/src/shr_RandNum_mod.F90
diff --git a/share/shr_RandNum/test/bench/Makefile b/src/share/RandNum/test/bench/Makefile
similarity index 100%
rename from share/shr_RandNum/test/bench/Makefile
rename to src/share/RandNum/test/bench/Makefile
diff --git a/share/shr_RandNum/test/bench/test_shr_RandNum.F90 b/src/share/RandNum/test/bench/test_shr_RandNum.F90
similarity index 100%
rename from share/shr_RandNum/test/bench/test_shr_RandNum.F90
rename to src/share/RandNum/test/bench/test_shr_RandNum.F90
diff --git a/share/esmf_wrf_timemgr/CMakeLists.txt b/src/share/esmf_wrf_timemgr/CMakeLists.txt
similarity index 100%
rename from share/esmf_wrf_timemgr/CMakeLists.txt
rename to src/share/esmf_wrf_timemgr/CMakeLists.txt
diff --git a/share/esmf_wrf_timemgr/ESMF.F90 b/src/share/esmf_wrf_timemgr/ESMF.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF.F90
rename to src/share/esmf_wrf_timemgr/ESMF.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_AlarmMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_BaseMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_BaseMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_BaseMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_BaseMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_CalendarMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_ClockMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_ClockMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_ClockMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_ClockMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_FractionMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_FractionMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_FractionMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_FractionMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_Macros.inc b/src/share/esmf_wrf_timemgr/ESMF_Macros.inc
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_Macros.inc
rename to src/share/esmf_wrf_timemgr/ESMF_Macros.inc
diff --git a/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_Stubs.F90 b/src/share/esmf_wrf_timemgr/ESMF_Stubs.F90
similarity index 79%
rename from share/esmf_wrf_timemgr/ESMF_Stubs.F90
rename to src/share/esmf_wrf_timemgr/ESMF_Stubs.F90
index 5deced65f09..9267c8fa80b 100644
--- a/share/esmf_wrf_timemgr/ESMF_Stubs.F90
+++ b/src/share/esmf_wrf_timemgr/ESMF_Stubs.F90
@@ -11,39 +11,47 @@ MODULE ESMF_Stubs
 ! Bogus typedefs
    TYPE ESMF_Grid
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_Grid
 
    TYPE ESMF_GridComp
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_GridComp
 
    TYPE ESMF_State
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_State
 
    TYPE ESMF_VM
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_VM
 
    TYPE ESMF_END_FLAG
       INTEGER :: dummy
-   END TYPE
-   TYPE(ESMF_END_FLAG), PARAMETER ::  &
-      ESMF_END_ABORT   = ESMF_END_FLAG(1), &
-      ESMF_END_NORMAL  = ESMF_END_FLAG(2), &
+   END TYPE ESMF_END_FLAG
+   TYPE(ESMF_END_FLAG), PARAMETER ::                &
+      ESMF_END_ABORT   = ESMF_END_FLAG(1),          &
+      ESMF_END_NORMAL  = ESMF_END_FLAG(2),          &
       ESMF_END_KEEPMPI = ESMF_END_FLAG(3)
 
    TYPE ESMF_MsgType
       INTEGER :: mtype
-   END TYPE
-   TYPE(ESMF_MsgType), PARAMETER  ::      &
-      ESMF_LOG_INFO  =   ESMF_MsgType(1), &
-      ESMF_LOG_WARNING = ESMF_MsgType(2), &
+   END TYPE ESMF_MsgType
+   TYPE(ESMF_MsgType), PARAMETER  ::                &
+      ESMF_LOG_INFO  =   ESMF_MsgType(1),           &
+      ESMF_LOG_WARNING = ESMF_MsgType(2),           &
       ESMF_LOG_ERROR =   ESMF_MsgType(3)
 
    TYPE ESMF_LOG
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_LOG
+
+   TYPE ESMF_LogKind_Flag
+      INTEGER :: dummy
+   END TYPE ESMF_LogKind_Flag
+   TYPE(ESMF_LogKind_Flag), PARAMETER ::            &
+        ESMF_LOGKIND_NONE = ESMF_LogKind_Flag(1),   &
+        ESMF_LOGKIND_SINGLE = ESMF_LogKind_Flag(2), &
+        ESMF_LOGKIND_MULTI = ESMF_LogKind_Flag(3)
 
    LOGICAL, private, save :: initialized = .false.
 
@@ -52,17 +60,20 @@ MODULE ESMF_Stubs
    PUBLIC ESMF_LogWrite, ESMF_LOG, ESMF_MsgType, ESMF_END_FLAG
    PUBLIC ESMF_LOG_INFO, ESMF_LOG_WARNING, ESMF_LOG_ERROR
    PUBLIC ESMF_END_ABORT, ESMF_END_NORMAL, ESMF_END_KEEPMPI
+   PUBLIC ESMF_LogKind_Flag
+   PUBLIC ESMF_LOGKIND_NONE, ESMF_LOGKIND_SINGLE, ESMF_LOGKIND_MULTI
 
 CONTAINS
 
 
 ! NOOP
-   SUBROUTINE ESMF_Initialize( vm, defaultCalendar, rc )
+   SUBROUTINE ESMF_Initialize( vm, defaultCalendar, logkindflag, rc )
       USE ESMF_BaseMod
       USE ESMF_CalendarMod
 !     USE ESMF_TimeMod,     only: defaultCal
       TYPE(ESMF_VM),           INTENT(IN   ), OPTIONAL :: vm
       TYPE(ESMF_CalKind_Flag), INTENT(IN   ), OPTIONAL :: defaultCalendar
+      TYPE(ESMF_LogKind_Flag), INTENT(IN   ), OPTIONAL :: logkindflag
       INTEGER,                 INTENT(  OUT), OPTIONAL :: rc
 
       TYPE(ESMF_CalKind_Flag) :: defaultCalType
diff --git a/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc b/src/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_TimeMgr.inc
rename to src/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc
diff --git a/share/esmf_wrf_timemgr/ESMF_TimeMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_TimeMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_TimeMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_TimeMod.F90
diff --git a/share/esmf_wrf_timemgr/Makefile b/src/share/esmf_wrf_timemgr/Makefile
similarity index 100%
rename from share/esmf_wrf_timemgr/Makefile
rename to src/share/esmf_wrf_timemgr/Makefile
diff --git a/share/esmf_wrf_timemgr/MeatMod.F90 b/src/share/esmf_wrf_timemgr/MeatMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/MeatMod.F90
rename to src/share/esmf_wrf_timemgr/MeatMod.F90
diff --git a/share/esmf_wrf_timemgr/README b/src/share/esmf_wrf_timemgr/README
similarity index 100%
rename from share/esmf_wrf_timemgr/README
rename to src/share/esmf_wrf_timemgr/README
diff --git a/share/esmf_wrf_timemgr/unittests/Makefile b/src/share/esmf_wrf_timemgr/unittests/Makefile
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/Makefile
rename to src/share/esmf_wrf_timemgr/unittests/Makefile
diff --git a/share/esmf_wrf_timemgr/unittests/go.csh b/src/share/esmf_wrf_timemgr/unittests/go.csh
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/go.csh
rename to src/share/esmf_wrf_timemgr/unittests/go.csh
diff --git a/share/esmf_wrf_timemgr/unittests/test.F90 b/src/share/esmf_wrf_timemgr/unittests/test.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/test.F90
rename to src/share/esmf_wrf_timemgr/unittests/test.F90
diff --git a/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90 b/src/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/wrf_stuff.F90
rename to src/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90
diff --git a/share/esmf_wrf_timemgr/wrf_error_fatal.F90 b/src/share/esmf_wrf_timemgr/wrf_error_fatal.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/wrf_error_fatal.F90
rename to src/share/esmf_wrf_timemgr/wrf_error_fatal.F90
diff --git a/share/esmf_wrf_timemgr/wrf_message.F90 b/src/share/esmf_wrf_timemgr/wrf_message.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/wrf_message.F90
rename to src/share/esmf_wrf_timemgr/wrf_message.F90
diff --git a/share/csm_share/include/dynamic_vector_procdef.inc b/src/share/include/dynamic_vector_procdef.inc
similarity index 100%
rename from share/csm_share/include/dynamic_vector_procdef.inc
rename to src/share/include/dynamic_vector_procdef.inc
diff --git a/share/csm_share/include/dynamic_vector_typedef.inc b/src/share/include/dynamic_vector_typedef.inc
similarity index 100%
rename from share/csm_share/include/dynamic_vector_typedef.inc
rename to src/share/include/dynamic_vector_typedef.inc
diff --git a/src/share/include/shr_assert.h b/src/share/include/shr_assert.h
new file mode 100644
index 00000000000..b09e0d12715
--- /dev/null
+++ b/src/share/include/shr_assert.h
@@ -0,0 +1,22 @@
+#ifdef NDEBUG
+#define SHR_ASSERT(assert, msg)
+#define SHR_ASSERT_FL(assert, file, line)
+#define SHR_ASSERT_MFL(assert, msg, file, line)
+#define SHR_ASSERT_ALL(assert, msg)
+#define SHR_ASSERT_ALL_FL(assert, file, line)
+#define SHR_ASSERT_ALL_MFL(assert, msg, file, line)
+#define SHR_ASSERT_ANY(assert, msg)
+#define SHR_ASSERT_ANY_FL(assert, file, line)
+#define SHR_ASSERT_ANY_MFL(assert, msg, file, line)
+#else
+#define SHR_ASSERT(assert, my_msg) call shr_assert(assert, msg=my_msg)
+#define SHR_ASSERT_FL(assert, my_file, my_line) call shr_assert(assert, file=my_file, line=my_line)
+#define SHR_ASSERT_MFL(assert, my_msg, my_file, my_line) call shr_assert(assert, msg=my_msg, file=my_file, line=my_line)
+#define SHR_ASSERT_ALL(assert, my_msg) call shr_assert_all(assert, msg=my_msg)
+#define SHR_ASSERT_ALL_FL(assert, my_file, my_line) call shr_assert_all(assert, file=my_file, line=my_line)
+#define SHR_ASSERT_ALL_MFL(assert, my_msg, my_file, my_line) call shr_assert_all(assert, msg=my_msg, file=my_file, line=my_line)
+#define SHR_ASSERT_ANY(assert, my_msg) call shr_assert_any(assert, msg=my_msg)
+#define SHR_ASSERT_ANY_FL(assert, my_file, my_line) call shr_assert_any(assert, file=my_file, line=my_line)
+#define SHR_ASSERT_ANY_MFL(assert, my_msg, my_file, my_line) call shr_assert_any(assert, msg=my_msg, file=my_file, line=my_line)
+#endif
+use shr_assert_mod
diff --git a/share/csm_share/test/old_unit_testers/Makefile b/src/share/test/old_unit_testers/Makefile
similarity index 100%
rename from share/csm_share/test/old_unit_testers/Makefile
rename to src/share/test/old_unit_testers/Makefile
diff --git a/share/csm_share/test/old_unit_testers/Mkdepends b/src/share/test/old_unit_testers/Mkdepends
similarity index 100%
rename from share/csm_share/test/old_unit_testers/Mkdepends
rename to src/share/test/old_unit_testers/Mkdepends
diff --git a/share/csm_share/test/old_unit_testers/Mksrcfiles b/src/share/test/old_unit_testers/Mksrcfiles
similarity index 100%
rename from share/csm_share/test/old_unit_testers/Mksrcfiles
rename to src/share/test/old_unit_testers/Mksrcfiles
diff --git a/share/csm_share/test/old_unit_testers/bundle_expected.F90 b/src/share/test/old_unit_testers/bundle_expected.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/bundle_expected.F90
rename to src/share/test/old_unit_testers/bundle_expected.F90
diff --git a/share/csm_share/test/old_unit_testers/config.h b/src/share/test/old_unit_testers/config.h
similarity index 100%
rename from share/csm_share/test/old_unit_testers/config.h
rename to src/share/test/old_unit_testers/config.h
diff --git a/share/csm_share/test/old_unit_testers/make.Macros b/src/share/test/old_unit_testers/make.Macros
similarity index 100%
rename from share/csm_share/test/old_unit_testers/make.Macros
rename to src/share/test/old_unit_testers/make.Macros
diff --git a/share/csm_share/test/old_unit_testers/namelist b/src/share/test/old_unit_testers/namelist
similarity index 100%
rename from share/csm_share/test/old_unit_testers/namelist
rename to src/share/test/old_unit_testers/namelist
diff --git a/share/csm_share/test/old_unit_testers/nl/atm.stdin b/src/share/test/old_unit_testers/nl/atm.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/atm.stdin
rename to src/share/test/old_unit_testers/nl/atm.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/cpl.stdin b/src/share/test/old_unit_testers/nl/cpl.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/cpl.stdin
rename to src/share/test/old_unit_testers/nl/cpl.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/ice.stdin b/src/share/test/old_unit_testers/nl/ice.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/ice.stdin
rename to src/share/test/old_unit_testers/nl/ice.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/lnd.stdin b/src/share/test/old_unit_testers/nl/lnd.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/lnd.stdin
rename to src/share/test/old_unit_testers/nl/lnd.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/ocn.stdin b/src/share/test/old_unit_testers/nl/ocn.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/ocn.stdin
rename to src/share/test/old_unit_testers/nl/ocn.stdin
diff --git a/share/csm_share/test/old_unit_testers/run_dshr_bundle_test b/src/share/test/old_unit_testers/run_dshr_bundle_test
similarity index 100%
rename from share/csm_share/test/old_unit_testers/run_dshr_bundle_test
rename to src/share/test/old_unit_testers/run_dshr_bundle_test
diff --git a/share/csm_share/test/old_unit_testers/run_file_test b/src/share/test/old_unit_testers/run_file_test
similarity index 100%
rename from share/csm_share/test/old_unit_testers/run_file_test
rename to src/share/test/old_unit_testers/run_file_test
diff --git a/share/csm_share/test/old_unit_testers/test_mod.F90 b/src/share/test/old_unit_testers/test_mod.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_mod.F90
rename to src/share/test/old_unit_testers/test_mod.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_file.F90 b/src/share/test/old_unit_testers/test_shr_file.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_file.F90
rename to src/share/test/old_unit_testers/test_shr_file.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_log.F90 b/src/share/test/old_unit_testers/test_shr_log.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_log.F90
rename to src/share/test/old_unit_testers/test_shr_log.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_mpi.F90 b/src/share/test/old_unit_testers/test_shr_mpi.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_mpi.F90
rename to src/share/test/old_unit_testers/test_shr_mpi.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_orb.F90 b/src/share/test/old_unit_testers/test_shr_orb.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_orb.F90
rename to src/share/test/old_unit_testers/test_shr_orb.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_scam.F90 b/src/share/test/old_unit_testers/test_shr_scam.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_scam.F90
rename to src/share/test/old_unit_testers/test_shr_scam.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_streams.F90 b/src/share/test/old_unit_testers/test_shr_streams.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_streams.F90
rename to src/share/test/old_unit_testers/test_shr_streams.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_sys.F90 b/src/share/test/old_unit_testers/test_shr_sys.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_sys.F90
rename to src/share/test/old_unit_testers/test_shr_sys.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_tInterp.F90 b/src/share/test/old_unit_testers/test_shr_tInterp.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_tInterp.F90
rename to src/share/test/old_unit_testers/test_shr_tInterp.F90
diff --git a/share/csm_share/test/unit/CMakeLists.txt b/src/share/test/unit/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/CMakeLists.txt
rename to src/share/test/unit/CMakeLists.txt
diff --git a/share/csm_share/test/unit/dynamic_vector/CMakeLists.txt b/src/share/test/unit/dynamic_vector/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/CMakeLists.txt
rename to src/share/test/unit/dynamic_vector/CMakeLists.txt
diff --git a/share/csm_share/test/unit/dynamic_vector/character16_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/character16_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/character16_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/character16_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc b/src/share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc
rename to src/share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_character16.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_character16.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_character16.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_character16.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_integer.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_integer.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_integer.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_integer.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_r8.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_r8.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_r8.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_r8.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/dynamic_vector/integer_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/integer_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/integer_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/integer_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/dynamic_vector/ptr_wrapper.F90 b/src/share/test/unit/dynamic_vector/ptr_wrapper.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/ptr_wrapper.F90
rename to src/share/test/unit/dynamic_vector/ptr_wrapper.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/r8_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/r8_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/r8_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/r8_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/mock/CMakeLists.txt b/src/share/test/unit/mock/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/mock/CMakeLists.txt
rename to src/share/test/unit/mock/CMakeLists.txt
diff --git a/share/csm_share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 b/src/share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90
similarity index 100%
rename from share/csm_share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90
rename to src/share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90
diff --git a/share/csm_share/test/unit/shr_assert_test/CMakeLists.txt b/src/share/test/unit/shr_assert_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_assert_test/CMakeLists.txt
rename to src/share/test/unit/shr_assert_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_assert_test/test_assert.pf b/src/share/test/unit/shr_assert_test/test_assert.pf
similarity index 70%
rename from share/csm_share/test/unit/shr_assert_test/test_assert.pf
rename to src/share/test/unit/shr_assert_test/test_assert.pf
index 5bc7239e26f..0e05b0c8a75 100644
--- a/share/csm_share/test/unit/shr_assert_test/test_assert.pf
+++ b/src/share/test/unit/shr_assert_test/test_assert.pf
@@ -22,9 +22,15 @@ end subroutine assert_can_pass
 @Test
 subroutine assert_can_fail()
   call shr_assert(.false., "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_can_fail
 
+@Test
+subroutine assert_prints_file_and_line()
+  call shr_assert(.false., "Expected failure.", file='foo', line=42)
+  call assertExceptionRaised("ABORTED: ERROR in foo at line 42: Expected failure.")
+end subroutine assert_prints_file_and_line
+
 @Test
 subroutine assert_all_scalar_can_pass()
   call shr_assert_all(.true., "Assert unexpectedly aborted!")
@@ -33,7 +39,7 @@ end subroutine assert_all_scalar_can_pass
 @Test
 subroutine assert_all_scalar_can_fail()
   call shr_assert_all(.false., "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_all_scalar_can_fail
 
 @Test
@@ -44,7 +50,7 @@ end subroutine assert_any_scalar_can_pass
 @Test
 subroutine assert_any_scalar_can_fail()
   call shr_assert_any(.false., "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_any_scalar_can_fail
 
 end module test_assert
diff --git a/share/csm_share/test/unit/shr_assert_test/test_assert_array.pf b/src/share/test/unit/shr_assert_test/test_assert_array.pf
similarity index 95%
rename from share/csm_share/test/unit/shr_assert_test/test_assert_array.pf
rename to src/share/test/unit/shr_assert_test/test_assert_array.pf
index 219b030a2aa..c847d4626a7 100644
--- a/share/csm_share/test/unit/shr_assert_test/test_assert_array.pf
+++ b/src/share/test/unit/shr_assert_test/test_assert_array.pf
@@ -73,7 +73,7 @@ subroutine assert_all_can_fail(this)
   class(TestAssertArray), intent(inout) :: this
   call assert_all_wrapper([.false.], 1, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_all_can_fail
 
 @Test
@@ -84,7 +84,7 @@ subroutine assert_all_partial_false_fails(this)
   test_array = [( mod(i,2) == 0, i = 1, size(test_array) )]
   call assert_all_wrapper(test_array, 2, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_all_partial_false_fails
 
 @Test
@@ -92,7 +92,7 @@ subroutine assert_any_size_zero_fails(this)
   class(TestAssertArray), intent(inout) :: this
   call assert_any_wrapper([logical::], 0, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_any_size_zero_fails
 
 @Test
@@ -107,7 +107,7 @@ subroutine assert_any_can_fail(this)
   class(TestAssertArray), intent(inout) :: this
   call assert_any_wrapper([.false.], 1, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_any_can_fail
 
 @Test
diff --git a/src/share/test/unit/shr_assert_test/test_macro.pf b/src/share/test/unit/shr_assert_test/test_macro.pf
new file mode 100644
index 00000000000..d75f39dae14
--- /dev/null
+++ b/src/share/test/unit/shr_assert_test/test_macro.pf
@@ -0,0 +1,87 @@
+module test_macro
+
+! Test that if NDEBUG is not defined, shr_assert macros run assertions.
+
+use pfunit_mod
+
+#undef NDEBUG
+#include "shr_assert.h"
+
+contains
+
+@Test
+subroutine macro_assert_can_pass()
+  SHR_ASSERT(.true., "Assert macro unexpectedly aborted!")
+end subroutine macro_assert_can_pass
+
+@Test
+subroutine macro_assert_can_fail()
+  SHR_ASSERT(.false., "Expected failure.")
+  ! When this was written, the preprocessor did not recognize this assert,
+  ! so call it directly instead of using an "@".
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
+end subroutine macro_assert_can_fail
+
+@Test
+subroutine macro_assert_fl()
+  SHR_ASSERT_FL(.false., "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42")
+end subroutine macro_assert_fl
+
+@Test
+subroutine macro_assert_mfl()
+  SHR_ASSERT_MFL(.false., "Expected failure.", "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42: Expected failure.")
+end subroutine macro_assert_mfl
+
+@Test
+subroutine macro_assert_all_can_pass()
+  SHR_ASSERT_ALL(([.true., .true.]), "Assert macro unexpectedly aborted!")
+end subroutine macro_assert_all_can_pass
+
+@Test
+subroutine macro_assert_all_can_fail()
+  SHR_ASSERT_ALL(([.true., .false.]), "Expected failure.")
+  ! When this was written, the preprocessor did not recognize this assert,
+  ! so call it directly instead of using an "@".
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
+end subroutine macro_assert_all_can_fail
+
+@Test
+subroutine macro_assert_all_fl()
+  SHR_ASSERT_ALL_FL(([.true., .false.]), "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42")
+end subroutine macro_assert_all_fl
+
+@Test
+subroutine macro_assert_all_mfl()
+  SHR_ASSERT_ALL_MFL(([.true., .false.]), "Expected failure.", "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42: Expected failure.")
+end subroutine macro_assert_all_mfl
+
+@Test
+subroutine macro_assert_any_can_pass()
+  SHR_ASSERT_ANY(([.true., .false.]), "Assert macro unexpectedly aborted!")
+end subroutine macro_assert_any_can_pass
+
+@Test
+subroutine macro_assert_any_can_fail()
+  SHR_ASSERT_ANY(([.false., .false.]), "Expected failure.")
+  ! When this was written, the preprocessor did not recognize this assert,
+  ! so call it directly instead of using an "@".
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
+end subroutine macro_assert_any_can_fail
+
+@Test
+subroutine macro_assert_any_fl()
+  SHR_ASSERT_ANY_FL(([.false., .false.]), "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42")
+end subroutine macro_assert_any_fl
+
+@Test
+subroutine macro_assert_any_mfl()
+  SHR_ASSERT_ANY_MFL(([.false., .false.]), "Expected failure.", "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42: Expected failure.")
+end subroutine macro_assert_any_mfl
+
+end module test_macro
diff --git a/src/share/test/unit/shr_assert_test/test_ndebug.pf b/src/share/test/unit/shr_assert_test/test_ndebug.pf
new file mode 100644
index 00000000000..ef1c6226447
--- /dev/null
+++ b/src/share/test/unit/shr_assert_test/test_ndebug.pf
@@ -0,0 +1,63 @@
+module test_ndebug
+
+! Test that if NDEBUG is defined, shr_assert macros do nothing.
+
+use pfunit_mod
+
+#define NDEBUG
+#include "shr_assert.h"
+
+contains
+
+  logical function unreachable_function(macro_name)
+    character(len=*), intent(in) :: macro_name
+
+    call throw("NDEBUG failed to turn off " // macro_name)
+  end function unreachable_function
+
+@Test
+subroutine ndebug_controls_assert_macro()
+  SHR_ASSERT(unreachable_function("SHR_ASSERT"), "Fake message.")
+end subroutine ndebug_controls_assert_macro
+
+@Test
+subroutine ndebug_controls_assert_fl_macro()
+  SHR_ASSERT_FL(unreachable_function("SHR_ASSERT_FL"), "my_file", 42)
+end subroutine ndebug_controls_assert_fl_macro
+
+@Test
+subroutine ndebug_controls_assert_mfl_macro()
+  SHR_ASSERT_MFL(unreachable_function("SHR_ASSERT_MFL"), "Fake message.", "my_file", 42)
+end subroutine ndebug_controls_assert_mfl_macro
+
+@Test
+subroutine ndebug_controls_assert_all_macro()
+  SHR_ASSERT_ALL(unreachable_function("SHR_ASSERT_ALL"), "Fake message.")
+end subroutine ndebug_controls_assert_all_macro
+
+@Test
+subroutine ndebug_controls_assert_all_fl_macro()
+  SHR_ASSERT_ALL_FL(unreachable_function("SHR_ASSERT_ALL_FL"), "my_file", 42)
+end subroutine ndebug_controls_assert_all_fl_macro
+
+@Test
+subroutine ndebug_controls_assert_all_mfl_macro()
+  SHR_ASSERT_ALL_MFL(unreachable_function("SHR_ASSERT_ALL_MFL"), "Fake message.", "my_file", 42)
+end subroutine ndebug_controls_assert_all_mfl_macro
+
+@Test
+subroutine ndebug_controls_assert_any_macro()
+  SHR_ASSERT_ANY(unreachable_function("SHR_ASSERT_ANY"), "Fake message.")
+end subroutine ndebug_controls_assert_any_macro
+
+@Test
+subroutine ndebug_controls_assert_any_fl_macro()
+  SHR_ASSERT_ANY_FL(unreachable_function("SHR_ASSERT_ANY_FL"), "my_file", 42)
+end subroutine ndebug_controls_assert_any_fl_macro
+
+@Test
+subroutine ndebug_controls_assert_any_mfl_macro()
+  SHR_ASSERT_ANY_MFL(unreachable_function("SHR_ASSERT_ANY_MFL"), "Fake message.", "my_file", 42)
+end subroutine ndebug_controls_assert_any_mfl_macro
+
+end module test_ndebug
diff --git a/share/csm_share/test/unit/shr_infnan_test/CMakeLists.txt b/src/share/test/unit/shr_infnan_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_infnan_test/CMakeLists.txt
rename to src/share/test/unit/shr_infnan_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_infnan_test/test_infnan.F90 b/src/share/test/unit/shr_infnan_test/test_infnan.F90
similarity index 100%
rename from share/csm_share/test/unit/shr_infnan_test/test_infnan.F90
rename to src/share/test/unit/shr_infnan_test/test_infnan.F90
diff --git a/share/csm_share/test/unit/shr_log_test/CMakeLists.txt b/src/share/test/unit/shr_log_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_log_test/CMakeLists.txt
rename to src/share/test/unit/shr_log_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_log_test/test_error_printers.pf b/src/share/test/unit/shr_log_test/test_error_printers.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_log_test/test_error_printers.pf
rename to src/share/test/unit/shr_log_test/test_error_printers.pf
diff --git a/share/csm_share/test/unit/shr_precip_test/CMakeLists.txt b/src/share/test/unit/shr_precip_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_precip_test/CMakeLists.txt
rename to src/share/test/unit/shr_precip_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_precip_test/test_shr_precip.pf b/src/share/test/unit/shr_precip_test/test_shr_precip.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_precip_test/test_shr_precip.pf
rename to src/share/test/unit/shr_precip_test/test_shr_precip.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/CMakeLists.txt b/src/share/test/unit/shr_spfn_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/CMakeLists.txt
rename to src/share/test/unit/shr_spfn_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_erf_r4.pf b/src/share/test/unit/shr_spfn_test/test_erf_r4.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_erf_r4.pf
rename to src/share/test/unit/shr_spfn_test/test_erf_r4.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_erf_r8.pf b/src/share/test/unit/shr_spfn_test/test_erf_r8.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_erf_r8.pf
rename to src/share/test/unit/shr_spfn_test/test_erf_r8.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_gamma_factorial.pf b/src/share/test/unit/shr_spfn_test/test_gamma_factorial.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_gamma_factorial.pf
rename to src/share/test/unit/shr_spfn_test/test_gamma_factorial.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_igamma.pf b/src/share/test/unit/shr_spfn_test/test_igamma.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_igamma.pf
rename to src/share/test/unit/shr_spfn_test/test_igamma.pf
diff --git a/share/csm_share/test/unit/shr_strconvert_test/CMakeLists.txt b/src/share/test/unit/shr_strconvert_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_strconvert_test/CMakeLists.txt
rename to src/share/test/unit/shr_strconvert_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_strconvert_test/test_toString.pf b/src/share/test/unit/shr_strconvert_test/test_toString.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_strconvert_test/test_toString.pf
rename to src/share/test/unit/shr_strconvert_test/test_toString.pf
diff --git a/share/csm_share/test/unit/shr_string_test/CMakeLists.txt b/src/share/test/unit/shr_string_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_string_test/CMakeLists.txt
rename to src/share/test/unit/shr_string_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_string_test/test_shr_string.pf b/src/share/test/unit/shr_string_test/test_shr_string.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_string_test/test_shr_string.pf
rename to src/share/test/unit/shr_string_test/test_shr_string.pf
diff --git a/share/csm_share/test/unit/shr_vmath_test/CMakeLists.txt b/src/share/test/unit/shr_vmath_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_vmath_test/CMakeLists.txt
rename to src/share/test/unit/shr_vmath_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_vmath_test/test_vmath.F90 b/src/share/test/unit/shr_vmath_test/test_vmath.F90
similarity index 100%
rename from share/csm_share/test/unit/shr_vmath_test/test_vmath.F90
rename to src/share/test/unit/shr_vmath_test/test_vmath.F90
diff --git a/share/csm_share/test/unit/shr_wv_sat_test/CMakeLists.txt b/src/share/test/unit/shr_wv_sat_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_wv_sat_test/CMakeLists.txt
rename to src/share/test/unit/shr_wv_sat_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat.pf b/src/share/test/unit/shr_wv_sat_test/test_wv_sat.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat.pf
rename to src/share/test/unit/shr_wv_sat_test/test_wv_sat.pf
diff --git a/share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf b/src/share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf
rename to src/share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf
diff --git a/share/timing/CMakeLists.txt b/src/share/timing/CMakeLists.txt
similarity index 100%
rename from share/timing/CMakeLists.txt
rename to src/share/timing/CMakeLists.txt
diff --git a/src/share/timing/COPYING b/src/share/timing/COPYING
new file mode 100644
index 00000000000..324ce86b24c
--- /dev/null
+++ b/src/share/timing/COPYING
@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software for any noncommercial purposes without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following
+conditions: The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.  Any
+commercial use (including sale) of the software, and derivative development
+towards commercial use, requires written permission of the copyright
+holder. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
+OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/share/timing/ChangeLog b/src/share/timing/ChangeLog
similarity index 100%
rename from share/timing/ChangeLog
rename to src/share/timing/ChangeLog
diff --git a/share/timing/GPTLget_memusage.c b/src/share/timing/GPTLget_memusage.c
similarity index 100%
rename from share/timing/GPTLget_memusage.c
rename to src/share/timing/GPTLget_memusage.c
diff --git a/share/timing/GPTLprint_memusage.c b/src/share/timing/GPTLprint_memusage.c
similarity index 100%
rename from share/timing/GPTLprint_memusage.c
rename to src/share/timing/GPTLprint_memusage.c
diff --git a/share/timing/GPTLutil.c b/src/share/timing/GPTLutil.c
similarity index 100%
rename from share/timing/GPTLutil.c
rename to src/share/timing/GPTLutil.c
diff --git a/share/timing/Makefile b/src/share/timing/Makefile
similarity index 100%
rename from share/timing/Makefile
rename to src/share/timing/Makefile
diff --git a/share/timing/README b/src/share/timing/README
similarity index 100%
rename from share/timing/README
rename to src/share/timing/README
diff --git a/share/timing/f_wrappers.c b/src/share/timing/f_wrappers.c
similarity index 100%
rename from share/timing/f_wrappers.c
rename to src/share/timing/f_wrappers.c
diff --git a/share/timing/gptl.c b/src/share/timing/gptl.c
similarity index 100%
rename from share/timing/gptl.c
rename to src/share/timing/gptl.c
diff --git a/share/timing/gptl.h b/src/share/timing/gptl.h
similarity index 100%
rename from share/timing/gptl.h
rename to src/share/timing/gptl.h
diff --git a/share/timing/gptl.inc b/src/share/timing/gptl.inc
similarity index 100%
rename from share/timing/gptl.inc
rename to src/share/timing/gptl.inc
diff --git a/share/timing/gptl_papi.c b/src/share/timing/gptl_papi.c
similarity index 100%
rename from share/timing/gptl_papi.c
rename to src/share/timing/gptl_papi.c
diff --git a/share/timing/perf_mod.F90 b/src/share/timing/perf_mod.F90
similarity index 100%
rename from share/timing/perf_mod.F90
rename to src/share/timing/perf_mod.F90
diff --git a/share/timing/perf_utils.F90 b/src/share/timing/perf_utils.F90
similarity index 100%
rename from share/timing/perf_utils.F90
rename to src/share/timing/perf_utils.F90
diff --git a/share/timing/private.h b/src/share/timing/private.h
similarity index 100%
rename from share/timing/private.h
rename to src/share/timing/private.h
diff --git a/share/csm_share/unit_test_stubs/README b/src/share/unit_test_stubs/README
similarity index 100%
rename from share/csm_share/unit_test_stubs/README
rename to src/share/unit_test_stubs/README
diff --git a/share/csm_share/unit_test_stubs/pio/CMakeLists.txt b/src/share/unit_test_stubs/pio/CMakeLists.txt
similarity index 100%
rename from share/csm_share/unit_test_stubs/pio/CMakeLists.txt
rename to src/share/unit_test_stubs/pio/CMakeLists.txt
diff --git a/share/csm_share/unit_test_stubs/pio/README b/src/share/unit_test_stubs/pio/README
similarity index 100%
rename from share/csm_share/unit_test_stubs/pio/README
rename to src/share/unit_test_stubs/pio/README
diff --git a/share/csm_share/unit_test_stubs/pio/pio.F90.in b/src/share/unit_test_stubs/pio/pio.F90.in
similarity index 95%
rename from share/csm_share/unit_test_stubs/pio/pio.F90.in
rename to src/share/unit_test_stubs/pio/pio.F90.in
index 22313cef88d..9aafa3550dc 100644
--- a/share/csm_share/unit_test_stubs/pio/pio.F90.in
+++ b/src/share/unit_test_stubs/pio/pio.F90.in
@@ -23,7 +23,7 @@ module pio
 #ifndef NO_MPIMOD
    use mpi, only : MPI_OFFSET_KIND ! _EXTERNAL
 #endif
-
+   use iso_c_binding
   implicit none
   private
 
@@ -43,7 +43,17 @@ module pio
   type, public :: var_desc_t
   end type var_desc_t
 
+   type, bind(c), public :: PIO_rearr_comm_fc_opt_t
+      logical(c_bool) :: enable_hs            ! Enable handshake?
+      logical(c_bool) :: enable_isend         ! Enable isends?
+      integer(c_int) :: max_pend_req         ! Maximum pending requests
+    end type PIO_rearr_comm_fc_opt_t
+
   type, public :: PIO_rearr_opt_t
+      integer(c_int)                         :: comm_type
+      integer(c_int)                         :: fcd       ! Flow control direction
+      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io
+      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp
   end type PIO_rearr_opt_t
 
   integer, parameter, private ::          &
@@ -70,6 +80,15 @@ module pio
   integer, parameter, public :: pio_rearr_box = 1
   integer, parameter, public :: PIO_WRITE = 0
 
+  integer, parameter, public :: pio_rearr_comm_p2p = 0
+  integer, parameter, public :: pio_rearr_comm_coll = 1
+  integer,parameter,public :: PIO_rearr_comm_fc_2d_enable = 0
+  integer,parameter,public :: PIO_rearr_comm_fc_1d_comp2io = 1
+  integer,parameter,public :: PIO_rearr_comm_fc_1d_io2comp = 2
+  integer,parameter,public :: PIO_rearr_comm_fc_2d_disable = 3
+  integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1
+
+
   public :: PIO_def_dim
   public :: PIO_enddef
   public :: PIO_FILE_IS_OPEN
@@ -344,16 +363,18 @@ contains
     ierr = 0
   end function get_var_vdesc_{DIMS}d_{TYPE}
 
-  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base)
+  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base, rearr_opts)
+
     integer(i4), intent(in) :: comp_rank
     integer(i4), intent(in) :: comp_comm
     integer(i4), intent(in) :: num_iotasks
     integer(i4), intent(in) :: num_aggregator
     integer(i4), intent(in) :: stride
     integer(i4), intent(in) :: rearr
-    type (iosystem_desc_t), intent(inout)  :: iosystem  ! io descriptor to initalize
-
+    type (iosystem_desc_t), intent(out)  :: iosystem  ! io descriptor to initalize
     integer(i4), intent(in),optional :: base
+    type (pio_rearr_opt_t), intent(in), optional :: rearr_opts
+
   end subroutine init_intracom
 
   subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosystem)
@@ -362,7 +383,7 @@ contains
     integer, intent(in) :: comp_comms(component_count)   !  The compute communicator
     integer, intent(in) :: io_comm     !  The io communicator
 
-    type (iosystem_desc_t), intent(inout)  :: iosystem(component_count)  ! io descriptor to initalize
+    type (iosystem_desc_t), intent(out)  :: iosystem(component_count)  ! io descriptor to initalize
   end subroutine init_intercom
 
   subroutine PIO_initdecomp_bc(iosystem,basepiotype,dims,compstart,compcount,iodesc,iostart,iocount)
diff --git a/share/csm_share/shr/CMakeLists.txt b/src/share/util/CMakeLists.txt
similarity index 100%
rename from share/csm_share/shr/CMakeLists.txt
rename to src/share/util/CMakeLists.txt
diff --git a/share/csm_share/shr/mct_mod.F90 b/src/share/util/mct_mod.F90
similarity index 99%
rename from share/csm_share/shr/mct_mod.F90
rename to src/share/util/mct_mod.F90
index 7f9dc3f6edb..80121dcfd49 100644
--- a/share/csm_share/shr/mct_mod.F90
+++ b/src/share/util/mct_mod.F90
@@ -1292,4 +1292,3 @@ end function mct_myindex
 !===============================================================================
 
 end module mct_mod
-
diff --git a/share/csm_share/shr/shr_assert_mod.F90.in b/src/share/util/shr_assert_mod.F90.in
similarity index 85%
rename from share/csm_share/shr/shr_assert_mod.F90.in
rename to src/share/util/shr_assert_mod.F90.in
index 1dce64b1157..fc62d64bae1 100644
--- a/share/csm_share/shr/shr_assert_mod.F90.in
+++ b/src/share/util/shr_assert_mod.F90.in
@@ -16,6 +16,8 @@ use shr_log_mod, only: &
 
 use shr_infnan_mod, only: shr_infnan_isnan
 
+use shr_strconvert_mod, only: toString
+
 implicit none
 private
 save
@@ -66,38 +68,64 @@ end interface
 
 contains
 
-subroutine shr_assert(var, msg)
+subroutine shr_assert(var, msg, file, line)
 
-  ! Logical being asserted.
+  ! Logical being asserted
   logical, intent(in) :: var
-  ! Optional error message if assert fails.
-  character(len=*), intent(in) :: msg
-
-  if (.not. var) call shr_sys_abort(msg)
+  ! Optional error message if assert fails
+  character(len=*), intent(in), optional :: msg
+  ! Optional file and line of the caller, written out if given
+  ! (line is ignored if file is absent)
+  character(len=*), intent(in), optional :: file
+  integer         , intent(in), optional :: line
+
+  character(len=:), allocatable :: full_msg
+
+  if (.not. var) then
+     full_msg = 'ERROR'
+     if (present(file)) then
+        full_msg = full_msg // ' in ' // trim(file)
+        if (present(line)) then
+           full_msg = full_msg // ' at line ' // toString(line)
+        end if
+     end if
+     if (present(msg)) then
+        full_msg = full_msg // ': ' // msg
+     end if
+     call shr_sys_abort(full_msg)
+  end if
 
 end subroutine shr_assert
 
 ! DIMS 1,2,3,4,5,6,7
-subroutine shr_assert_all_{DIMS}d(var, msg)
+subroutine shr_assert_all_{DIMS}d(var, msg, file, line)
 
-  ! Logical being asserted.
+  ! Logical being asserted
   logical, intent(in) :: var{DIMSTR}
-  ! Optional error message if assert fails.
-  character(len=*), intent(in) :: msg
+  ! Optional error message if assert fails
+  character(len=*), intent(in), optional :: msg
+  ! Optional file and line of the caller, written out if given
+  ! (line is ignored if file is absent)
+  character(len=*), intent(in), optional :: file
+  integer         , intent(in), optional :: line
 
-  call shr_assert(all(var), msg)
+  call shr_assert(all(var), msg=msg, file=file, line=line)
 
 end subroutine shr_assert_all_{DIMS}d
 
 ! DIMS 1,2,3,4,5,6,7
-subroutine shr_assert_any_{DIMS}d(var, msg)
+subroutine shr_assert_any_{DIMS}d(var, msg, file, line)
 
-  ! Logical being asserted.
+  ! Logical being asserted
   logical, intent(in) :: var{DIMSTR}
-  ! Optional error message if assert fails.
-  character(len=*), intent(in) :: msg
+  ! Optional error message if assert fails
+  character(len=*), intent(in), optional :: msg
+  ! Optional file and line of the caller, written out if given
+  ! (line is ignored if file is absent)
+  character(len=*), intent(in), optional :: file
+  integer         , intent(in), optional :: line
 
-  call shr_assert(any(var), msg)
+  call shr_assert(any(var), msg=msg, file=file, line=line)
 
 end subroutine shr_assert_any_{DIMS}d
 
diff --git a/share/csm_share/shr/shr_cal_mod.F90 b/src/share/util/shr_cal_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_cal_mod.F90
rename to src/share/util/shr_cal_mod.F90
diff --git a/share/csm_share/shr/shr_const_mod.F90 b/src/share/util/shr_const_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_const_mod.F90
rename to src/share/util/shr_const_mod.F90
diff --git a/share/csm_share/shr/shr_dmodel_mod.F90 b/src/share/util/shr_dmodel_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_dmodel_mod.F90
rename to src/share/util/shr_dmodel_mod.F90
diff --git a/share/csm_share/shr/shr_file_mod.F90 b/src/share/util/shr_file_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_file_mod.F90
rename to src/share/util/shr_file_mod.F90
diff --git a/share/csm_share/shr/shr_flux_mod.F90 b/src/share/util/shr_flux_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_flux_mod.F90
rename to src/share/util/shr_flux_mod.F90
diff --git a/share/csm_share/shr/shr_frz_mod.F90.in b/src/share/util/shr_frz_mod.F90.in
similarity index 100%
rename from share/csm_share/shr/shr_frz_mod.F90.in
rename to src/share/util/shr_frz_mod.F90.in
diff --git a/share/csm_share/shr/shr_infnan_mod.F90.in b/src/share/util/shr_infnan_mod.F90.in
similarity index 100%
rename from share/csm_share/shr/shr_infnan_mod.F90.in
rename to src/share/util/shr_infnan_mod.F90.in
diff --git a/share/csm_share/shr/shr_kind_mod.F90 b/src/share/util/shr_kind_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_kind_mod.F90
rename to src/share/util/shr_kind_mod.F90
diff --git a/share/csm_share/shr/shr_log_mod.F90 b/src/share/util/shr_log_mod.F90
similarity index 88%
rename from share/csm_share/shr/shr_log_mod.F90
rename to src/share/util/shr_log_mod.F90
index 230cf8aa889..e116df57e09 100644
--- a/share/csm_share/shr/shr_log_mod.F90
+++ b/src/share/util/shr_log_mod.F90
@@ -53,6 +53,13 @@ module shr_log_mod
 !     \newline
 !     errMsg = shr\_log\_errMsg(__FILE__, __LINE__)
 !
+! This is meant to be used when a routine expects a string argument for some message,
+! but you want to provide file and line information.
+!
+! However: Note that the performance of this function can be very bad. It is currently
+! maintained because it is used by old code, but you should probably avoid using this
+! in new code if possible.
+!
 ! !REVISION HISTORY:
 !     2013-July-23 - Bill Sacks
 !
diff --git a/share/csm_share/shr/shr_map_mod.F90 b/src/share/util/shr_map_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_map_mod.F90
rename to src/share/util/shr_map_mod.F90
diff --git a/share/csm_share/shr/shr_mct_mod.F90 b/src/share/util/shr_mct_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_mct_mod.F90
rename to src/share/util/shr_mct_mod.F90
diff --git a/share/csm_share/shr/shr_mem_mod.F90 b/src/share/util/shr_mem_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_mem_mod.F90
rename to src/share/util/shr_mem_mod.F90
diff --git a/share/csm_share/shr/shr_mpi_mod.F90 b/src/share/util/shr_mpi_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_mpi_mod.F90
rename to src/share/util/shr_mpi_mod.F90
diff --git a/share/csm_share/shr/shr_msg_mod.F90 b/src/share/util/shr_msg_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_msg_mod.F90
rename to src/share/util/shr_msg_mod.F90
diff --git a/share/csm_share/shr/shr_ncread_mod.F90 b/src/share/util/shr_ncread_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_ncread_mod.F90
rename to src/share/util/shr_ncread_mod.F90
diff --git a/share/csm_share/shr/shr_nl_mod.F90 b/src/share/util/shr_nl_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_nl_mod.F90
rename to src/share/util/shr_nl_mod.F90
diff --git a/share/csm_share/shr/shr_orb_mod.F90 b/src/share/util/shr_orb_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_orb_mod.F90
rename to src/share/util/shr_orb_mod.F90
diff --git a/share/csm_share/shr/shr_pcdf_mod.F90 b/src/share/util/shr_pcdf_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_pcdf_mod.F90
rename to src/share/util/shr_pcdf_mod.F90
diff --git a/share/csm_share/shr/shr_pio_mod.F90 b/src/share/util/shr_pio_mod.F90
similarity index 98%
rename from share/csm_share/shr/shr_pio_mod.F90
rename to src/share/util/shr_pio_mod.F90
index f4030f0a584..2e02ed83c52 100644
--- a/share/csm_share/shr/shr_pio_mod.F90
+++ b/src/share/util/shr_pio_mod.F90
@@ -196,7 +196,6 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia
              call shr_pio_read_component_namelist(nlfilename , comp_comm(i), pio_comp_settings(i)%pio_stride, &
                   pio_comp_settings(i)%pio_root, pio_comp_settings(i)%pio_numiotasks, &
                   pio_comp_settings(i)%pio_iotype, pio_comp_settings(i)%pio_rearranger)
-#ifdef PIO1
              call pio_init(comp_comm_iam(i), comp_comm(i), pio_comp_settings(i)%pio_numiotasks, 0, &
                   pio_comp_settings(i)%pio_stride, &
                   pio_comp_settings(i)%pio_rearranger, iosystems(i), &
@@ -207,14 +206,6 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia
                 write(shr_log_unit,*) io_compname(i),' : pio_root = ',pio_comp_settings(i)%pio_root
                 write(shr_log_unit,*) io_compname(i),' : pio_iotype = ',pio_comp_settings(i)%pio_iotype
              end if
-#else
-             call pio_init(comp_comm_iam(i), comp_comm(i), pio_comp_settings(i)%pio_numiotasks, 0, &
-                  pio_comp_settings(i)%pio_stride, &
-                  pio_comp_settings(i)%pio_rearranger, iosystems(i), &
-                  base=pio_comp_settings(i)%pio_root)
-#endif
-
-
           end if
        end do
     end if
@@ -447,13 +438,11 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,
     call shr_mpi_bcast(pio_async_interface, Comm)
     call shr_mpi_bcast(pio_rearranger, Comm)
 
-#ifdef PIO1
      call shr_pio_rearr_opts_set(Comm, pio_rearr_comm_type, pio_rearr_comm_fcd, &
            pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
            pio_rearr_comm_enable_isend_comp2io, &
            pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
            pio_rearr_comm_enable_isend_io2comp, pio_numiotasks)
-#endif
 
   end subroutine shr_pio_read_default_namelist
 
@@ -645,7 +634,7 @@ subroutine shr_pio_namelist_set(npes,mycomm, pio_stride, pio_root, pio_numiotask
     end if
 
   end subroutine shr_pio_namelist_set
-#ifdef PIO1
+
   ! This subroutine sets the global PIO rearranger options
   ! The input args that represent the rearranger options are valid only
   ! on the root proc of comm
@@ -818,7 +807,6 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
       pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .true.
     end if
   end subroutine
-#endif
 !===============================================================================
 
 end module shr_pio_mod
diff --git a/share/csm_share/shr/shr_precip_mod.F90 b/src/share/util/shr_precip_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_precip_mod.F90
rename to src/share/util/shr_precip_mod.F90
diff --git a/share/csm_share/shr/shr_reprosum_mod.F90 b/src/share/util/shr_reprosum_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_reprosum_mod.F90
rename to src/share/util/shr_reprosum_mod.F90
diff --git a/share/csm_share/shr/shr_reprosumx86.c b/src/share/util/shr_reprosumx86.c
similarity index 100%
rename from share/csm_share/shr/shr_reprosumx86.c
rename to src/share/util/shr_reprosumx86.c
diff --git a/share/csm_share/shr/shr_scam_mod.F90 b/src/share/util/shr_scam_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_scam_mod.F90
rename to src/share/util/shr_scam_mod.F90
diff --git a/share/csm_share/shr/shr_spfn_mod.F90 b/src/share/util/shr_spfn_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_spfn_mod.F90
rename to src/share/util/shr_spfn_mod.F90
diff --git a/share/csm_share/shr/shr_strconvert_mod.F90 b/src/share/util/shr_strconvert_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_strconvert_mod.F90
rename to src/share/util/shr_strconvert_mod.F90
diff --git a/share/csm_share/shr/shr_strdata_mod.F90 b/src/share/util/shr_strdata_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_strdata_mod.F90
rename to src/share/util/shr_strdata_mod.F90
diff --git a/share/csm_share/shr/shr_stream_mod.F90 b/src/share/util/shr_stream_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_stream_mod.F90
rename to src/share/util/shr_stream_mod.F90
diff --git a/share/csm_share/shr/shr_string_mod.F90 b/src/share/util/shr_string_mod.F90
similarity index 99%
rename from share/csm_share/shr/shr_string_mod.F90
rename to src/share/util/shr_string_mod.F90
index 428f9ac881d..f50898227bc 100644
--- a/share/csm_share/shr/shr_string_mod.F90
+++ b/src/share/util/shr_string_mod.F90
@@ -1732,7 +1732,7 @@ function shr_string_listCreateField( numFields, strBase ) result ( retString )
    ! this assert isn't that accurate since it counts all integers as being one
    ! digit, but it should catch most errors and under rather than overestimates
    !
-   SHR_ASSERT( ( ( ( len(strBase) + 3 ) * numFields ) <= 1024 ) , errMsg(__FILE__, __LINE__) )
+   SHR_ASSERT_FL( ( ( ( len(strBase) + 3 ) * numFields ) <= 1024 ) , __FILE__, __LINE__)
 
    retString = ''
    do idx = 1,numFields
diff --git a/share/csm_share/shr/shr_sys_mod.F90 b/src/share/util/shr_sys_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_sys_mod.F90
rename to src/share/util/shr_sys_mod.F90
diff --git a/share/csm_share/shr/shr_tInterp_mod.F90 b/src/share/util/shr_tInterp_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_tInterp_mod.F90
rename to src/share/util/shr_tInterp_mod.F90
diff --git a/share/csm_share/shr/shr_timer_mod.F90 b/src/share/util/shr_timer_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_timer_mod.F90
rename to src/share/util/shr_timer_mod.F90
diff --git a/share/csm_share/shr/shr_vmath_mod.F90 b/src/share/util/shr_vmath_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_vmath_mod.F90
rename to src/share/util/shr_vmath_mod.F90
diff --git a/share/csm_share/shr/shr_wv_sat_mod.F90 b/src/share/util/shr_wv_sat_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_wv_sat_mod.F90
rename to src/share/util/shr_wv_sat_mod.F90
diff --git a/share/csm_share/shr/water_isotopes.F90 b/src/share/util/water_isotopes.F90
similarity index 100%
rename from share/csm_share/shr/water_isotopes.F90
rename to src/share/util/water_isotopes.F90
diff --git a/share/csm_share/shr/water_types.F90 b/src/share/util/water_types.F90
similarity index 100%
rename from share/csm_share/shr/water_types.F90
rename to src/share/util/water_types.F90
diff --git a/tools/Readme.configure b/tools/Readme.configure
index 6e1058e6472..36b310ee5f9 100644
--- a/tools/Readme.configure
+++ b/tools/Readme.configure
@@ -1,36 +1,60 @@
-SYNOPSIS
-     configure [options]
-OPTIONS
-     User supplied values are denoted in angle brackets (<>).  Any value that contains
-     white-space must be quoted.  Long option names may be supplied with either single
-     or double leading dashes.  A consequence of this is that single letter options may
-     NOT be bundled.
+usage: configure [-h] [-d] [-v] [-s] [--machine MACHINE]
+                 [--machines-dir MACHINES_DIR]
+                 [--macros-format {Makefile,CMake}] [--output-dir OUTPUT_DIR]
+                 [--compiler COMPILER] [--mpilib MPILIB]
 
-     -cimeroot            Specify the toplevel cime directory.
-                          default: use CIMEROOT environment variable
-     -mach          Specify a machine (required).
-     -compiler      Specify a compiler for the target machine (optional)
-                          default: default compiler for the target machine
-     -mpilib        Specify an mpi library for the target machine (optional)
-                          default: mpi-serial  (most of the tools are serial apps)
-     -mach_dir      Specify the locations of the Machines directory (optional).
+This script writes CIME build information to a directory. The pieces of
+information that will be written include: 1. Machine-specific build settings
+(i.e. the "Macros" file). 2. File-specific build settings (i.e. "Depends"
+files). 3. Environment variable loads (i.e. the env_mach_specific files). The
+.env_mach_specific.sh and .env_mach_specific.csh files are specific to a given
+compiler, MPI library, and DEBUG setting. By default, these will be the
+machine's default compiler, the machine's default MPI library, and FALSE,
+respectively. These can be changed by setting the environment variables
+COMPILER, MPILIB, and DEBUG, respectively.
 
-     -output_dir    default: current working directory
-     -output_format  Output format can be make or cmake.
-                          default: make
-     -help [or -h]        Print usage to STDOUT (optional).
-     -loglevel
+optional arguments:
+  -h, --help            show this help message and exit
+  -d, --debug           Print debug information (very verbose) to file /home/j
+                        edwards/cesm2_0_alpha/cime/tools/mapping/map_field/src
+                        /configure.log
+  -v, --verbose         Add additional context (time and file) to log messages
+  -s, --silent          Print only warnings and error messages
+  --machine MACHINE     The machine to create build information for.
+  --machines-dir MACHINES_DIR
+                        The machines directory to take build information from.
+                        Overrides the CIME_MODEL environment variable, and
+                        must be specified if that variable is not set.
+  --macros-format {Makefile,CMake}
+                        The format of Macros file to generate. If 'Makefile'
+                        is passed in, a file called 'Macros.make' is
+                        generated. If 'CMake' is passed in, a file called
+                        'Macros.cmake' is generated. This option can be
+                        specified multiple times to generate multiple files.
+                        If not used at all, Macros generation is skipped. Note
+                        that Depends files are currently always in Makefile
+                        format, regardless of this option.
+  --output-dir OUTPUT_DIR
+                        The directory to write files to. If not specified,
+                        defaults to the current working directory.
+  --compiler COMPILER, -compiler COMPILER
+                        Specify a compiler. To see list of supported compilers
+                        for each machine, use the utility manage_case in this
+                        directory
+  --mpilib MPILIB, -mpilib MPILIB
+                        Specify the mpilib. To see list of supported mpilibs
+                        for each machine, use the utility manage_case in this
+                        directory. The default is the first listing in MPILIBS
+                        in config_machines.xml
+  --clean               Remove old Macros and env files before attempting to
+                        create new ones
 
-EXAMPLES
 
-  ./configure -mach bluefire -compiler ibm -cimeroot /path/to/cime
 
-
-This tool is used to create Macros files to build CESM tools on supported machines.
 It will create a Macros file in make or cmake format along with the module support
 for the specified or default compiler.   It will also create two files
 .env_mach_specific.csh
 .env_mach_specific.sh
 
 Before you try to run make you should source the file above appropriate for your shell
-this will set the proper environment for supported systems which use modules.
\ No newline at end of file
+this will set the proper environment for supported systems which use modules.
diff --git a/tools/configure b/tools/configure
index 3632f599fca..bba14b82d70 100755
--- a/tools/configure
+++ b/tools/configure
@@ -20,7 +20,7 @@ import sys
 
 _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
 sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools"))
-sys.path.append(os.path.join(_CIMEROOT, "scripts", "utils", "python"))
+sys.path.append(os.path.join(_CIMEROOT, "scripts", "lib"))
 
 from standard_script_setup import *
 from CIME.utils import expect, get_model
@@ -55,6 +55,19 @@ def parse_command_line(args):
                         help="The directory to write files to. If not "
                         "specified, defaults to the current working directory.")
 
+    parser.add_argument("--compiler", "-compiler",
+                        help="Specify a compiler. "
+                        "To see list of supported compilers for each machine, use the utility manage_case in this directory")
+
+    parser.add_argument("--mpilib", "-mpilib",
+                        help="Specify the mpilib. "
+                        "To see list of supported mpilibs for each machine, use the utility manage_case in this directory. "
+                        "The default is the first listing in MPILIBS in config_machines.xml")
+
+    parser.add_argument("--clean", action="store_true", 
+                        help="Remove old Macros and env files before attempting to create new ones")
+
+    argcnt = len(args)
     args = parser.parse_args()
     CIME.utils.handle_standard_logging_options(args)
 
@@ -83,7 +96,9 @@ def parse_command_line(args):
     opts['output_dir'] = args.output_dir
 
     # Set compiler.
-    if "COMPILER" in os.environ:
+    if args.compiler is not None:
+        compiler = args.compiler
+    elif "COMPILER" in os.environ:
         compiler = os.environ["COMPILER"]
     else:
         compiler = machobj.get_default_compiler()
@@ -94,11 +109,14 @@ def parse_command_line(args):
     opts['compiler'] = compiler
     opts['os'] = machobj.get_value('OS')
     # Set MPI library.
-    if "MPILIB" in os.environ:
+    if args.mpilib is not None:
+        mpilib = args.mpilib
+    elif "MPILIB" in os.environ:
         mpilib = os.environ["MPILIB"]
     else:
         mpilib = machobj.get_default_MPIlib()
         os.environ["MPILIB"] = mpilib
+
     expect(opts['machobj'].is_valid_MPIlib(mpilib),
            "Invalid MPI library name given in MPILIB environment variable: %s" %
            mpilib)
@@ -115,12 +133,25 @@ def parse_command_line(args):
         os.environ["DEBUG"] = "FALSE"
     opts['debug'] = debug
 
+
+    if args.clean:
+        files = ["Macros.make", "Macros.cmake", "env_mach_specific.xml", ".env_mach_specific.sh", 
+                 ".env_mach_specific.csh", "Depends.%s"%compiler, "Depends.%s"%args.machine,
+                  "Depends.%s.%s"%(args.machine,compiler)]
+        for file_ in files:
+            if os.path.isfile(file_):
+                logger.warn("Removing file %s"%file_)
+                os.remove(file_)
+        if argcnt == 2:
+            opts['clean_only'] = True
+
     return opts
 
 def _main():
     opts = parse_command_line(sys.argv)
-    configure(opts['machobj'], opts['output_dir'], opts['macros_format'],
-              opts['compiler'], opts['mpilib'], opts['debug'], opts['os'])
+    if "clean_only" not in opts or not opts["clean_only"]:
+        configure(opts['machobj'], opts['output_dir'], opts['macros_format'],
+                  opts['compiler'], opts['mpilib'], opts['debug'], opts['os'])
 
 if __name__ == "__main__":
     _main()
diff --git a/tools/cprnc/CMakeLists.txt b/tools/cprnc/CMakeLists.txt
index f923e6bbd3b..cdff9be18e0 100644
--- a/tools/cprnc/CMakeLists.txt
+++ b/tools/cprnc/CMakeLists.txt
@@ -1,7 +1,8 @@
 PROJECT(CPRNC C Fortran)
 
+ 	
 execute_process(COMMAND ../configure --macros-format=CMake WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "$ENV{CIMEROOT}/externals/CMake")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "$ENV{CIMEROOT}/src/externals/CMake")
 
 INCLUDE(Macros.cmake)
 
@@ -9,7 +10,7 @@ ENABLE_LANGUAGE(Fortran)
 
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
-SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} $ENV{CIMEROOT}/externals/pio2/cmake)
+SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} $ENV{CIMEROOT}/src/externals/pio2/cmake)
 
 # Defining NETCDF_LIBRARIES and NETCDF_INCLUDE_DIR enables bypassing
 #   the search for Netcdf
@@ -20,7 +21,7 @@ ENDIF ()
 
 ADD_CUSTOM_COMMAND(
   OUTPUT ${PROJECT_BINARY_DIR}/compare_vars_mod.F90
-  COMMAND perl ${PROJECT_SOURCE_DIR}/../../externals/genf90/genf90.pl
+  COMMAND perl ${PROJECT_SOURCE_DIR}/../../src/externals/genf90/genf90.pl
     ${PROJECT_SOURCE_DIR}/compare_vars_mod.F90.in > ${PROJECT_BINARY_DIR}/compare_vars_mod.F90
 )
 
diff --git a/tools/cprnc/Makefile b/tools/cprnc/Makefile
index 3957f8e14b7..ba5940d1e26 100644
--- a/tools/cprnc/Makefile
+++ b/tools/cprnc/Makefile
@@ -41,7 +41,7 @@ INC_NETCDF := $(NETCDF_PATH)/include
 UNAMES := $(shell uname -s)
 SNAME := $(shell uname -n | cut -c1-2)
 
-GENF90 = ../../externals/genf90/genf90.pl
+GENF90 = ../../src/externals/genf90/genf90.pl
 
 FC := $(SFC)
 FFLAGS += -I$(INC_NETCDF) -I.
diff --git a/tools/cprnc/summarize_cprnc_diffs b/tools/cprnc/summarize_cprnc_diffs
index cb8a3c8f830..1d6b5cf8d22 100755
--- a/tools/cprnc/summarize_cprnc_diffs
+++ b/tools/cprnc/summarize_cprnc_diffs
@@ -53,7 +53,8 @@ if (!$opts{'testid'}) {
 #    Dir => directory (gives test name)
 #    Filename  => cprnc filename
 #    Variable  => variable
-#    RMS       => normalized rms value [may or may not be present]
+#    RMS       => rms value            [may or may not be present]
+#    RMS_NORM  => normalized rms value [may or may not be present]
 #    FILLDIFF  => ' '                  [may or may not be present]
 #    DIMSIZEDIFF => ' '                [may or may not be present]
 my ($summary_hash) =
@@ -106,7 +107,7 @@ SYNOPSIS
     it only looks at output for baseline comparisons - NOT output from the test
     itself, such as cprnc output files from the exact restart test.)
 
-    Summaries of cprnc differences (normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs)
+    Summaries of cprnc differences (RMS and normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs)
     are placed in three output files beginning with the name 'cprnc.summary', in
     the current directory.  These files contain the same information, but one is
     sorted by test name, one is sorted by variable name, and is one sorted from
@@ -127,7 +128,7 @@ EOF
 
 
 # process_cprnc_output
-# Read through all cprnc files, and build hashes of instances of RMS, FILLDIFF and DIMSIZEDIFF
+# Read through all cprnc files, and build hashes of instances of RMS, normalized RMS, FILLDIFF and DIMSIZEDIFF
 # Inputs:
 #  - basedir
 #  - testid
@@ -214,13 +215,13 @@ sub process_line {
 
       # For RMS errors, keep the highest error found
       if ($diff_type eq "RMS") {
-         if (exists $diffs->{$key} && exists $diffs->{$key}{$diff_type}) {
-            if ($diffs->{$key}{$diff_type} > $rms_normalized) {
-               warn "WARNING: Ignoring lower RMS value:        $key : $rms_normalized < $diffs->{$key}{$diff_type}\n";
+         if (exists $diffs->{$key} && exists $diffs->{$key}{'RMS_NORM'}) {
+            if ($diffs->{$key}{'RMS_NORM'} > $rms_normalized) {
+               warn "WARNING: Ignoring lower RMS value:        $key : $rms_normalized < $diffs->{$key}{'RMS_NORM'}\n";
                return;
             }
             else {
-               warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{$diff_type}\n";
+               warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{'RMS_NORM'}\n";
             }
          }
       }
@@ -237,11 +238,16 @@ sub process_line {
       }
 
       # Whether or not the hash already contained the given key, we need to add
-      # the value of interest -- either the normalized RMS error or the fact
-      # that there is a FILLDIFF or DIMSIZEDIFF; in the latter cases, note that
-      # $rms_normalized is irrelevant, but we use it as the value of the hash
-      # anyway for simplicity.
-      $diffs->{$key}{$diff_type} = $rms_normalized;
+      # the value of interest -- either the RMS and normalized RMS errors, or
+      # the fact that there is a FILLDIFF or DIMSIZEDIFF.
+      if ($diff_type eq "RMS") {
+         $diffs->{$key}{'RMS'} = $rms;
+         $diffs->{$key}{'RMS_NORM'} = $rms_normalized;
+      } else {
+         # No meaningful value here - just record the fact that we saw a
+         # FILLDIFF or DIMSIZEDIFF
+         $diffs->{$key}{$diff_type} = "";
+      }
    } elsif ($diff_type ne '') {
       die "Unexpected diff_type: $diff_type";
    }
@@ -287,7 +293,7 @@ sub print_results_by_test {
    my $last_dir;
    my $last_filename;
 
-   my $separator_width = sum(values %$widths) + 30;
+   my $separator_width = sum(values %$widths) + 57;
 
    for my $key (@sorted_keys) {
 
@@ -327,7 +333,7 @@ sub print_results_by_varname {
 
    my $last_variable;
 
-   my $separator_width = sum(values %$widths) + 30;
+   my $separator_width = sum(values %$widths) + 57;
 
    for my $key (@sorted_keys) {
 
@@ -349,7 +355,7 @@ sub print_results_by_varname {
 
 
 
-# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS
+# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS_NORM
 # Inputs:
 #  - outfile: name of output file
 #  - summary_hash: hash reference containing results to print
@@ -359,7 +365,7 @@ sub print_results_by_rms {
 
    open OUT, ">", "$outfile" or die "ERROR opening $outfile";
 
-   my @sorted_keys = sort {$summary_hash->{$b}{'RMS'}      <=> $summary_hash->{$a}{'RMS'}
+   my @sorted_keys = sort {$summary_hash->{$b}{'RMS_NORM'} <=> $summary_hash->{$a}{'RMS_NORM'}
                         or $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                         or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'}
                         or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} }
@@ -381,6 +387,7 @@ sub print_results_by_rms {
 #   - Filename
 #   - Variable
 #   - RMS (optional)
+#   - RMS_NORM (optional)
 #   - FILLDIFF (optional)
 #   - DIMSIZEDIFF (optional)
 # - widths: hash reference giving widths of output strings
@@ -392,11 +399,15 @@ sub format_line {
    my $filename = $hash_ref->{'Filename'};
    my $variable = $hash_ref->{'Variable'};
    my $rms = "";
+   my $rms_normalized = "";
    my $filldiff = "";
    my $dimsizediff = "";
    if (exists $hash_ref->{'RMS'}) {
       $rms = sprintf(" : RMS %-16g", $hash_ref->{'RMS'});
    }
+   if (exists $hash_ref->{'RMS_NORM'}) {
+      $rms_normalized = sprintf(" : RMS_NORM %-16g", $hash_ref->{'RMS_NORM'});
+   }
    if (exists $hash_ref->{'FILLDIFF'}) {
       $filldiff = " : FILLDIFF";
    }
@@ -408,9 +419,9 @@ sub format_line {
    my $format = '%-' . $widths->{'Dir'} . '.' . $widths->{'Dir'} . 's : ' .
                 '%-' . $widths->{'Filename'} . '.' . $widths->{'Filename'} . 's : ' .
                 '%-' . $widths->{'Variable'} . '.' . $widths->{'Variable'} . 's' .
-                '%s%s%s';
+                '%s%s%s%s';
 
-   sprintf($format, $dir, $filename, $variable, $filldiff, $dimsizediff, $rms);
+   sprintf($format, $dir, $filename, $variable, $filldiff, $dimsizediff, $rms, $rms_normalized);
 }
 
 #=======================================================================
@@ -432,10 +443,10 @@ sub format_line {
 # process_line("FILLDIFF var1", "test_dir1", "file_b", \%diffs);
 
 # # add an RMS to existing filldiff
-# process_line("RMS var1 99 NORMALIZED 42", "test_dir1", "file_b", \%diffs);
+# process_line("RMS var1 4200 NORMALIZED 42", "test_dir1", "file_b", \%diffs);
 
 # # test basic rms error
-# process_line("RMS var17 99 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs);
+# process_line("RMS var17 0.314 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs);
 
 # # add a filldiff to existing rms error
 # process_line("FILLDIFF var17", "test_dir1", "file_b", \%diffs);
@@ -450,55 +461,60 @@ sub format_line {
 # process_line("RMS var100 99 NORMALIZED 100", "test_dir2", "file_d", \%diffs);
 
 # # test a warning: should issue a warning and replace the above setting
-# process_line("RMS var100 99 NORMALIZED 200", "test_dir2", "file_d", \%diffs);
+# process_line("RMS var100 9 NORMALIZED 200", "test_dir2", "file_d", \%diffs);
 
 # # test a warning: should issue a warning but NOT replace the above setting
-# process_line("RMS var100 99 NORMALIZED 50", "test_dir2", "file_d", \%diffs);
+# # (normalized RMS is smaller even though standard RMS is bigger: the normalized
+# # one should be considered in deciding whether to replace the previous setting)
+# process_line("RMS var100 999 NORMALIZED 50", "test_dir2", "file_d", \%diffs);
 
 # print Dumper(\%diffs);
 
 
-# THE ABOVE SHOULD PRINT:
+# THE ABOVE SHOULD PRINT SOMETHING LIKE THIS (though the output from Dumper will
+# likely appear in a different order):
 
 # WARNING: Replacing RMS with higher value: test_dir2 file_d var100 : 200 > 100
 # WARNING: Ignoring lower RMS value:        test_dir2 file_d var100 : 50 < 200
 # $VAR1 = {
 #           'test_dir1 file_b var17' => {
-#                                         'FILLDIFF' => '',
-#                                         'RMS' => '3.14',
-#                                         'Filename' => 'file_b',
+#                                         'RMS' => '0.314',
 #                                         'Variable' => 'var17',
-#                                         'Dir' => 'test_dir1'
-#                                       },
-#           'test_dir2 file_c var42' => {
+#                                         'Filename' => 'file_b',
 #                                         'FILLDIFF' => '',
-#                                         'Filename' => 'file_c',
-#                                         'Variable' => 'var42',
-#                                         'Dir' => 'test_dir2'
+#                                         'Dir' => 'test_dir1',
+#                                         'RMS_NORM' => '3.14'
 #                                       },
-#           'test_dir1 file_b var1' => {
-#                                        'FILLDIFF' => '',
-#                                        'RMS' => '42',
-#                                        'Filename' => 'file_b',
-#                                        'Variable' => 'var1',
-#                                        'Dir' => 'test_dir1'
-#                                      },
 #           'test_dir2 file_d var100' => {
-#                                          'RMS' => 200,
+#                                          'Dir' => 'test_dir2',
+#                                          'RMS_NORM' => 200,
 #                                          'Filename' => 'file_d',
 #                                          'Variable' => 'var100',
-#                                          'Dir' => 'test_dir2'
-#                                        }
+#                                          'RMS' => '9'
+#                                        },
+#           'test_dir1 file_b var1' => {
+#                                        'Filename' => 'file_b',
+#                                        'RMS_NORM' => '42',
+#                                        'FILLDIFF' => '',
+#                                        'Dir' => 'test_dir1',
+#                                        'RMS' => '4200',
+#                                        'Variable' => 'var1'
+#                                      },
 #           'test_dir2 file_c var43' => {
-#                                          'Variable' => 'var43',
-#                                          'DIMSIZEDIFF' => '',
-#                                          'Dir' => 'test_dir2',
-#                                          'Filename' => 'file_c'
-#                                        }
+#                                         'Variable' => 'var43',
+#                                         'DIMSIZEDIFF' => '',
+#                                         'Dir' => 'test_dir2',
+#                                         'Filename' => 'file_c'
+#                                       },
+#           'test_dir2 file_c var42' => {
+#                                         'Filename' => 'file_c',
+#                                         'Dir' => 'test_dir2',
+#                                         'FILLDIFF' => '',
+#                                         'Variable' => 'var42'
+#                                       }
 #         };
 
 
-
 #-----------------------------------------------------------------------
 # Testing the print routines
 #-----------------------------------------------------------------------
@@ -512,19 +528,19 @@ sub format_line {
 # This should give:
 
 # $ cat testout.by_rms
-# test_dir2                                : file_d                                   : var100                                   : RMS 200
-# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 42
-# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 3.14
+# test_dir2                                : file_d                                   : var100                                   : RMS 9                : RMS_NORM 200
+# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 4200             : RMS_NORM 42
+# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 0.314            : RMS_NORM 3.14
 # test_dir2                                : file_c                                   : var42                                    : FILLDIFF
 # test_dir2                                : file_c                                   : var43                                    : DIMSIZEDIFF
 # $ cat testout.by_test
-# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 42
-# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 3.14
-# ======================================================================================================================================================
+# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 4200             : RMS_NORM 42
+# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 0.314            : RMS_NORM 3.14
+# =================================================================================================================================================================================
 # test_dir2                                : file_c                                   : var42                                    : FILLDIFF
 # test_dir2                                : file_c                                   : var43                                    : DIMSIZEDIFF
-# ======================================================================================================================================================
-# test_dir2                                : file_d                                   : var100                                   : RMS 200
+# =================================================================================================================================================================================
+# test_dir2                                : file_d                                   : var100                                   : RMS 9                : RMS_NORM 200
 
 
 
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL b/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL
index 098bcb55443..d57d7b49545 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL
@@ -2,17 +2,16 @@
 HOW TO BUILD
 ============
 
-Prior to building, you must make sure $CIMEROOT is set.
-
 (1) $ cd src
-(2) $ $CIMEROOT/tools/configure -mach [machine name]
-(3) Bash users:
-    $ . .env_mach_specific.sh
-    csh users:
-    $ source .env_mach_specific.csh
-(4) $ gmake
+(2) $ ../../../../configure --macros-format Makefile --mpilib mpi-serial
+Bash users:
+(3) $ (. .env_mach_specific.sh ; gmake)
+csh users:
+(3) $ (source .env_mach_specific.csh ; gmake)
 
-Note: in the second step, replace [machine name] with the machine you are
-building on. Also, some machines have dedicated build nodes, so you might need
-to SSH to another node before the 'gmake' step.
+Note: in the second step, you may need to include "--machine [machine name]",
+where [machine name] is the name of the machine you are building on. In most
+cases configure can figure that out on its own, but if you get an error that is
+the first fix to try. Also, some machines have dedicated build nodes, so you
+might need to SSH to another node before the 'gmake' step.
 
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/README b/tools/mapping/gen_mapping_files/runoff_to_ocn/README
index fe37d86ba3d..fe6a1f54e77 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/README
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/README
@@ -1,86 +1,159 @@
-===============
-ABOUT THIS TOOL
-===============
+=================
+ABOUT THESE TOOLS
+=================
 
-The src/ directory here contains F90 files and a Makefile to produce the
-runoff_map executable, which reads in a runoff file and an ocean grid file
-and outputs a map from the former to the latter:
+The CIME coupler can map two different types of runoff to the ocean: liquid
+(river) and ice; these tools are provided to help generate both maps. The src/
+directory contains F90 files and a Makefile to produce the runoff_map executable
+(see the INSTALL file for more details), and run_merge_mapping_files.sh is a
+bash script that provides a clean interface to an NCL script.
 
-1) Computes an initial nearest neighbor mapping from the rof grid cells to
-   active ocean grid cells.
-2) Computes a smoothing matrix on the ocean grid.
-3) The two matrices are multipled together to generate the final mapping file.
+======================
+HOW TO USE THESE TOOLS
+======================
 
-====================
-HOW TO USE THIS TOOL
-====================
+1) Build (see the INSTALL file in this directory for details)
+2) Setup a namelist file (see the "NAMELIST FILE FORMAT" section below or any
+   runoff_map_*.nml for an example)
+3) Run with the following command:
 
-1) Build (in the src/ directory)
-2) Setup a namelist file (runoff_map.nml)
-3) Run ./runoff_map
+$ ./runoff_map < [namelist file]
 
-============
-HOW TO BUILD
-============
+4) For some ocean models, it is useful to use combine two liquid runoff maps
+   so that river runoff sent to the open ocean is treated differently than
+   runoff sent to a marginal sea. This step is generally not needed for ice
+   runoff, but two maps can be merged with the following command:
+   
+$ ./run_merge_mapping_files.sh
 
-See INSTALL file in this directory
+See the "MERGE MAPPING FILES" section below for more details.
 
 ====================
 NAMELIST FILE FORMAT
 ====================
 
-The namelist file must be called "runoff_map.nml".  That name is hardwired
-into the executable.
-
 The namelist looks like this
 
-  &input_nml
-   gridtype     = 'scrip'
-   file_roff    = '/glade/p/cesm/cseg/mapping/grids/wr50a_090614.nc'
-   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/ar9v4_100920.nc'
-   file_nn      = 'map_wr50a_ar9v4_nn.nc '
-   file_smooth  = 'map_ar9v4_ar9v4_smoother.nc '
-   file_new     = 'map_wr50a_to_ar9v4_e1000r300_130507.nc'
-   title        = 'runoff map: wr50a -> ar9v4, smoothed '
+&input_nml
+   gridtype     = 'rtm'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/clm2/rtmdata/rdirc.05.061026'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx1v7_151008.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx1v7_coast_170322.nc'
+   file_nn      = 'map_r05_to_gx1v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx1v7_coast_to_gx1v7_sm_e1000r300_170324.nc'
+   file_new     = 'map_r05_to_gx1v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: r05 -> gx1v7, nearest neighbor and smoothed'
    eFold        = 1000000.0
    rMax         =  300000.0
+   restrict_smooth_src_to_nn_dest = .true.
    step1 = .true.
-   step2 = .false.
-   step3 = .false.
-  /
-
-where
-
-Input grid files:
-  file_roff =   an ascii rdirc file OR an obs rtm file OR a scrip grid file
-  gridtype  =  type of file_roff file, "rtm" or "obs" or "scrip"
-     rtm is a 720 x 360 grid ascii file
-     obs is a netcdf file with xc, yc, xv, yv, mask and area variable names
-     scrip is a scrip type grid file (must contain grid_area along with typical
-       scrip grid variables)
-  file_ocn  =  a scrip ocean grid file (must contain grid_area along with
-    typical scrip grid variables)
-
-Input parameters:
-  eFold  = smoothing eFold distance in meters
-  rMax   = maximum radius of effect in meters
-
-Settings:
-  title = ascii string to add to mapping files
-  step1 = computes nearest neighbor map
-  step2 = computes smooth map
-  step3 = multiple two maps together
-
-Output fields:
-  file_nn     = nearest neighbor mapping file
-  file_smooth = smoother mapping file
-  file_new    = combined file
-
-
-==========
-HOW TO RUN
-==========
-
-Execute the binary ./runoff_map (again, this will read the runoff_map.nml
-namelist file - see above section for tips on creating the file)
+   step2 = .true.
+   step3 = .true.
+/
+
+Where the variables can be divided into four categories:
+
+1. Input grid files
+  gridtype =  type of file_roff file, "rtm" or "obs" or "scrip"
+              * rtm is a 720 x 360 grid ascii file
+              * obs is a netcdf file with xc, yc, xv, yv, mask and area variable names
+              * scrip is a scrip type grid file (must contain grid_area along with
+                typical scrip grid variables)
+  file_roff  =  an ascii rdirc file OR an obs rtm file OR a scrip grid file
+  file_ocn   =  a scrip ocean grid file where the mask is 1 for all ocean grid
+                cells (see note 3 below)
+  file_ocn_coastal_mask  =  a scrip ocean grid file where the mask is only 1
+                            for coastal grid cells (see note 3 below)
+
+NOTES:
+1) gridtype, file_roff, and file_ocn MUST be specified in the namelist
+2) if file_ocn_coastal_mask is not specified, file_ocn will be used
+3) The file_ocn and file_ocn_coast_mask must be standard scrip grid files that
+   include the cell area
+
+2. Input parameters
+  eFold = smoothing eFold distance in meters (default: 1000000)
+  rMax  = maximum radius of effect in meters (default: 300000)
+
+3. Settings
+  title                 = ascii string to add to mapping files (default: 'unset')
+  restrict_smooth_src_to_nn_dest = option to limit the source points for step2 to
+                                   just the points that get mapped to in step1; if
+                                   false, use all ocean points in
+                                   file_ocn_coastal_mask instead (default: .true.)
+  step1                 = computes nearest neighbor map (default: .true.)
+  step2                 = computes smooth map (default: .true.)
+  step3                 = multiply two maps together (default: .true.)
+
+4. Output fields
+  file_nn     = nearest neighbor mapping file (default: 'nn.nc')
+  file_smooth = smoother mapping file (default: 'smooth.nc')
+  file_new    = combined file (default: 'nnsm.nc')
+
+========================
+WHAT THESE TOOLS PROVIDE
+========================
+
+The runoff_map executable generates three maps:
+
+1) A nearest neighbor mapping from the rof grid to the ocean grid.
+   * For river runoff, it may be desireble to limit the destination grid so that
+     runoff only enters at coastal points. The optional file_ocn_coast_mask
+     variable in the namelist allows you to specify two ocean grid files; one for
+     the full ocean and one for just the coast. Typically this should not be
+     specified for ice runoff.
+2) A smoothing map from the ocean grid to the ocean grid.
+   * If file_ocn_coast_mask was specified, this maps from the coastal ocean to
+     the open ocean; otherwise it maps from the ocean grid onto itself.
+   * By default, the source domain of this map is only cells that appear as
+     destination cells of the nearest neighbor map. Optionally, one can
+     set restrict_smooth_src_to_nn_dest = .false. and generate a map that maps
+     from all ocean cells, even if they are not destination cells of the nearest
+     neighbor map. Enabling this map will not change the resulting map, but will
+     allow the resulting map to be reused if you are generating multiple maps
+     from different runoff grids onto the same ocean grid.
+   * If file_ocn_coast_mask is not specified, be aware that setting
+     restrict_smooth_src_to_nn_dest = .false. will take significantly longer
+     (and generate a much larger file) than leaving it .true.
+3) The product of the previous two maps.
+
+The ncl/ directory contains an NCL script to compute a fourth map (which is
+actually a combination of two existing maps):
+
+4) Map (1) from above for rof grid cells that map into the open ocean, or map
+   (3) for the rof grid cells that map into a marginal sea.
+   * This is only necessary if runoff should be treated differently in the open
+     ocean than in marginal seas, otherwise map (3) from above should be used
+     and this step is not needed.
+
+===================
+MERGE MAPPING FILES
+===================
+
+The run_merge_mapping_files.sh script requires four arguments:
+
+$ ./run_merge_mapping_files --map_in_oo MAP_IN_OO_FNAME     \
+                            --map_in_ms MAP_IN_MS_FNAME     \
+                            --region_mask REGION_MASK_FNAME \
+                            --map_out MAP_OUT_FNAME         \
+                            [-h -v]
+
+  -h,--help     show this message
+  -v,--verbose  echo file names back to screen before running NCL script
+  --map_in_oo MAP_IN_OO_FNAME
+                mapping file containing map to open ocean points
+  --map_in_ms MAP_IN_MS_FNAME
+                mapping file containing map to marginal sea points
+  --region_mask REGION_MASK_FNAME
+                POP region mask file (to specify open ocean vs marginal sea)
+  --map_out MAP_OUT_FNAME
+                output file
+
+A couple of useful notes:
+* MAP_IN_OO_FNAME and MAP_IN_MS_FNAME are typically maps (1) and (3),
+  respectively, from the "WHAT THESE TOOLS PROVIDE" section.
+* REGION_MASK_FNAME is a binary file (written with big endian convention)
+  containing one integer per grid cell -- a positive number represents a region
+  in open ocean, a negative number represents a region in a marginal sea, and a
+  zero is land.
 
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..174d39af5f9
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'rtm'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/clm2/rtmdata/rdirc.05.061026'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx1v7_151008.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx1v7_coast_170322.nc'
+   file_nn      = 'map_r05_to_gx1v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx1v7_coast_to_gx1v7_sm_e1000r300_170324.nc'
+   file_new     = 'map_r05_to_gx1v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: r05 -> gx1v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml
new file mode 100644
index 00000000000..d1c72a7382f
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'rtm'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/clm2/rtmdata/rdirc.05.061026'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx3v7_120309.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx3v7_coast_161214.nc'
+   file_nn      = 'map_r05_to_gx3v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx3v7_coast_to_gx3v7_sm_e1000r500_170324.nc'
+   file_new     = 'map_r05_to_gx3v7_nnsm_e1000r500_170324.nc'
+   title        = 'runoff map: r05 -> gx3v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  500000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..dbbd6a0081f
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'obs'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/dlnd7/RX1/runoff.daitren.annual.090225.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx1v7_151008.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx1v7_coast_170322.nc'
+   file_nn      = 'map_rx1_to_gx1v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx1v7_coast_to_gx1v7_sm_e1000r300_170324a.nc'
+   file_new     = 'map_rx1_to_gx1v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: rx1 -> gx1v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml
new file mode 100644
index 00000000000..9345b5f2199
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'obs'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/dlnd7/RX1/runoff.daitren.annual.090225.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx3v7_120309.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx3v7_coast_161214.nc'
+   file_nn      = 'map_rx1_to_gx3v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx3v7_coast_to_gx3v7_sm_e1000r500_170324a.nc'
+   file_new     = 'map_rx1_to_gx3v7_nnsm_e1000r500_170324.nc'
+   title        = 'runoff map: rx1 -> gx3v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  500000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..2f8fe7a4c21
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml
@@ -0,0 +1,14 @@
+&input_nml
+   gridtype     = 'scrip'
+   file_roff    = '/glade/p/cesm/cseg/mapping/grids/wr50a_090614.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/ar9v4_100920.nc'
+   file_nn      = 'map_wr50a_to_ar9v4_nearestdtos_170324.nc'
+   file_smooth  = 'map_ar9v4_to_ar9v4_sm_e1000r300_170324.nc'
+   file_new     = 'map_wr50a_to_ar9v4_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: wr50a -> ar9v4, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..29a2f9ec33a
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml
@@ -0,0 +1,14 @@
+&input_nml
+   gridtype     = 'scrip'
+   file_roff    = '/glade/p/cesm/cseg/mapping/grids/wr50a_090614.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx3v7_120309.nc'
+   file_nn      = 'map_wr50a_to_gx3v7_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx3v7_to_gx3v7_sm_e1000r300_170324.nc'
+   file_new     = 'map_wr50a_to_gx3v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: wr50a -> gx3v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore b/tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore
new file mode 100644
index 00000000000..d57580fd328
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore
@@ -0,0 +1 @@
+*.nc
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl b/tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl
new file mode 100644
index 00000000000..1e113bd1a25
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl
@@ -0,0 +1,210 @@
+;
+; merge_mapping_files.ncl
+;
+; merge two mapping files into one
+;
+; Keith Lindsay, 2017-03-08
+; Mike Levy, 2017-03-23 (updated how arguments are passed to this script)
+;
+;   3 options for running this program:
+;
+;     1) use the ../run_merge_mapping_files.sh wrapper script
+;        $ ../run_merge_mapping_files.sh --map_in_oo MAP_IN_OO_FNAME     \
+;                                        --map_in_ms MAP_IN_MS_FNAME     \
+;                                        --region_mask REGION_MASK_FNAME \
+;                                        --map_out MAP_OUT_FNAME
+;
+;     2) ncl merge_mapping_files.ncl 'MAP_IN_OO_FNAME="file1_name"'   \
+;                                    'MAP_IN_MS_FNAME="file2_name"'   \
+;                                    'REGION_MASK_FNAME="file3_name"' \
+;                                    'MAP_OUT_FNAME="file4_name"'
+;
+;     3) Set following environment variables:
+;        MAP_IN_OO_FNAME (entries into the open ocean)
+;        MAP_IN_MS_FNAME (entries into marginal seas)
+;        REGION_MASK_FNAME (marginal seas designation)
+;        MAP_OUT_FNAME (output file)
+;
+
+begin
+  ; Initialize variables from command line or environment
+  if (.not. isvar("MAP_IN_OO_FNAME")) then
+     MAP_IN_OO_FNAME   = getenv("MAP_IN_OO_FNAME")
+  end if
+
+  if (.not. isvar("MAP_IN_MS_FNAME")) then
+     MAP_IN_MS_FNAME   = getenv("MAP_IN_MS_FNAME")
+  end if
+
+  if (.not. isvar("REGION_MASK_FNAME")) then
+     REGION_MASK_FNAME   = getenv("REGION_MASK_FNAME")
+  end if
+
+  if (.not. isvar("MAP_OUT_FNAME")) then
+     MAP_OUT_FNAME   = getenv("MAP_OUT_FNAME")
+  end if
+
+  ; Make sure all input files exist
+  missing_files = False
+  if (.not. fileexists(MAP_IN_OO_FNAME)) then
+    print((/"ERROR: can not find open ocean map ("+MAP_IN_OO_FNAME+ ")"/))
+    missing_files = True
+  end if
+  if (.not. fileexists(MAP_IN_MS_FNAME)) then
+    print((/"ERROR: can not find marginal sea map ("+MAP_IN_MS_FNAME+ ")"/))
+    missing_files = True
+  end if
+  if (.not. fileexists(REGION_MASK_FNAME)) then
+    print((/"ERROR: can not find region mask ("+REGION_MASK_FNAME+ ")"/))
+    missing_files = True
+  end if
+
+  if (missing_files) then
+    status_exit(1)
+  end if
+
+  map_in_oo_fp      = addfile(MAP_IN_OO_FNAME, "r")
+  map_in_ms_fp      = addfile(MAP_IN_MS_FNAME, "r")
+
+  ; ensure grid definitions in MAP_IN_MS_FNAME agrees with grid definitions in MAP_IN_MS_FNAME
+  print("comparing grid vars in")
+  print("  MAP_IN_OO_FNAME = " + MAP_IN_OO_FNAME)
+  print("  MAP_IN_MS_FNAME = " + MAP_IN_MS_FNAME)
+  grid_varnames = (/ "xc_a", "yc_a", "mask_a", "area_a", "frac_a", "src_grid_dims", \
+                     "xc_b", "yc_b",           "area_b",           "dst_grid_dims" /)
+  diffs_found = False
+  do varind = 0, dimsizes(grid_varnames)-1
+    varname = grid_varnames(varind)
+    field_oo := map_in_oo_fp->$varname$
+    field_ms := map_in_ms_fp->$varname$
+    if (.not. all(ismissing(field_oo) .eq. ismissing(field_ms))) then
+      print("  " + vars1(var_ind) + " _FillValue patterns differ")
+      diffs_found = True
+    end if
+    if (all(field_oo .eq. field_ms)) then
+      print("  " + varname + " values agree")
+    else
+      print("  " + varname + " values differ")
+      if (any(abs(field_oo - field_ms) .gt. 1e-13 * (abs(field_oo)>abs(field_ms)))) then
+        print("    differences appear to be greater than round-off")
+        diffs_found = True
+      else
+        print("    differences appear to be less than round-off")
+      end if
+    end if
+  end do
+  if (.not. diffs_found) then
+    print("grid variables are compatible")
+  else
+    status_exit(1)
+  end if
+
+  print("reading grid info and map from " + MAP_IN_OO_FNAME)
+  xc_a          = map_in_oo_fp->xc_a
+  yc_a          = map_in_oo_fp->yc_a
+  xc_b          = map_in_oo_fp->xc_b
+  yc_b          = map_in_oo_fp->yc_b
+  dst_grid_dims = map_in_oo_fp->dst_grid_dims
+  map_in_oo_S   = map_in_oo_fp->S
+  map_in_oo_col = map_in_oo_fp->col ; 1-based indices
+  map_in_oo_row = map_in_oo_fp->row ; 1-based indices
+  n_s_oo        = dimsizes(map_in_oo_S)
+
+  print("reading map from " + MAP_IN_MS_FNAME)
+  map_in_ms_S   = map_in_ms_fp->S
+  map_in_ms_col = map_in_ms_fp->col
+  map_in_ms_row = map_in_ms_fp->row
+  n_s_ms        = dimsizes(map_in_ms_S)
+
+  ; ensure that maps have same unique col values, i.e., they are mapping from same src points
+  print("checking col value compatibility")
+  map_in_oo_col_unique = get_unique_values(map_in_oo_col)
+  map_in_ms_col_unique = get_unique_values(map_in_ms_col)
+  if (dimsizes(map_in_oo_col_unique) .ne. dimsizes(map_in_ms_col_unique)) then
+    print("number of unique col values in " + MAP_IN_OO_FNAME + " and " + MAP_IN_MS_FNAME + " differ")
+    status_exit(1)
+  end if
+  if (any(map_in_oo_col_unique .ne. map_in_ms_col_unique)) then
+    print("unique col values in " + MAP_IN_OO_FNAME + " and " + MAP_IN_MS_FNAME + " differ")
+    status_exit(1)
+  end if
+
+  ; load POP region mask from REGION_MASK_FNAME
+  print("reading REGION_MASK")
+  setfileoption("bin", "ReadByteOrder", "BigEndian")
+  REGION_MASK = cbinread(REGION_MASK_FNAME, (/ dst_grid_dims(1), dst_grid_dims(0) /), "integer")
+  REGION_MASK_1d = ndtooned(REGION_MASK)
+
+  ; create merged mapping
+
+  ; determine size of merged map
+
+  print("determining map_oo vals where REGION_MASK>0")
+  sign_REGION_MASK_oo = sign_matlab(REGION_MASK_1d(map_in_oo_row-1))
+  ind_vals_oo = ind(sign_REGION_MASK_oo .gt. 0)
+  n_s_subset_oo = dimsizes(ind_vals_oo)
+
+  print("determining map_ms vals where REGION_MASK<0")
+  sign_REGION_MASK_ms = sign_matlab(REGION_MASK_1d(map_in_ms_row-1))
+  ind_vals_ms = ind(sign_REGION_MASK_ms .lt. 0)
+  n_s_subset_ms = dimsizes(ind_vals_ms)
+
+  n_s_out = n_s_subset_oo + n_s_subset_ms
+
+  ; allocate output vars
+
+  map_out_S = new(n_s_out, double)
+  map_out_S!0 = "n_s"
+  delete(map_out_S@_FillValue)
+  map_out_S@long_name = map_in_ms_S@long_name
+
+  map_out_col = new(n_s_out, integer)
+  map_out_col!0 = "n_s"
+  delete(map_out_col@_FillValue)
+  map_out_col@long_name = map_in_ms_col@long_name
+
+  map_out_row = new(n_s_out, integer)
+  map_out_row!0 = "n_s"
+  delete(map_out_row@_FillValue)
+  map_out_row@long_name = map_in_ms_row@long_name
+
+  ; fill output vars
+
+  map_out_S(0:n_s_subset_oo-1)   = (/ map_in_oo_S(ind_vals_oo) /)
+  map_out_col(0:n_s_subset_oo-1) = (/ map_in_oo_col(ind_vals_oo) /)
+  map_out_row(0:n_s_subset_oo-1) = (/ map_in_oo_row(ind_vals_oo) /)
+
+  map_out_S(n_s_subset_oo:n_s_subset_oo+n_s_subset_ms-1)   = (/ map_in_ms_S(ind_vals_ms) /)
+  map_out_col(n_s_subset_oo:n_s_subset_oo+n_s_subset_ms-1) = (/ map_in_ms_col(ind_vals_ms) /)
+  map_out_row(n_s_subset_oo:n_s_subset_oo+n_s_subset_ms-1) = (/ map_in_ms_row(ind_vals_ms) /)
+
+  print("writing merged map, " + MAP_OUT_FNAME)
+  system("rm -f " + MAP_OUT_FNAME)
+  map_out_fp = addfile(MAP_OUT_FNAME, "c")
+
+  map_out_fp@history = "File created: "+systemfunc("date +%Y-%m-%d\ %H:%M:%S")
+  map_out_fp@conventions = "NCAR-CCSM"
+  map_out_fp@open_ocean_mapping_fname = MAP_IN_OO_FNAME
+  map_out_fp@marginal_seas_mapping_fname = MAP_IN_MS_FNAME
+  map_out_fp@region_mask_fname = REGION_MASK_FNAME
+
+  copy_varnames = (/ "xc_a", "yc_a", "xv_a", "yv_a", "mask_a", "area_a", "frac_a", "src_grid_dims", \
+                     "xc_b", "yc_b", "xv_b", "yv_b", "mask_b", "area_b", "frac_b", "dst_grid_dims" /)
+
+  do varind = 0, dimsizes(copy_varnames)-1
+    varname = copy_varnames(varind)
+    map_out_fp->$varname$ = map_in_ms_fp->$varname$
+  end do
+
+  ; read mask from map_ms (for size and metadata)
+  ; set it to 1 where merged mapping maps to
+  mask_b = map_in_ms_fp->mask_b
+  mask_b = 0
+  mask_b(map_out_row-1) = 1
+  map_out_fp->mask_b = (/ mask_b /)
+
+  filedimdef(map_out_fp, "n_s", n_s_out, False)
+  map_out_fp->S   = map_out_S(0:n_s_out-1)
+  map_out_fp->col = map_out_col(0:n_s_out-1)
+  map_out_fp->row = map_out_row(0:n_s_out-1)
+end
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh b/tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh
new file mode 100755
index 00000000000..8fc5133fe7b
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+#
+
+################################################################################
+
+main() {
+  parse_args $@
+  check_inputs
+  if [ ! -z $VERBOSE ]; then
+    print_summary
+    echo "Running ncl script..."
+    echo ""
+  fi
+
+  ncl ncl/merge_mapping_files.ncl
+}
+
+################################################################################
+
+usage() {
+  cat << EOF
+$ ./run_merge_mapping_files --map_in_oo MAP_IN_OO_FNAME     \\
+                            --map_in_ms MAP_IN_MS_FNAME     \\
+                            --region_mask REGION_MASK_FNAME \\
+                            --map_out MAP_OUT_FNAME         \\
+                            [-h -v]
+
+  -h,--help     show this message
+  -v,--verbose  echo file names back to screen before running NCL script
+  --map_in_oo MAP_IN_OO_FNAME
+                mapping file containing map to open ocean points
+  --map_in_ms MAP_IN_MS_FNAME
+                mapping file containing map to marginal sea points
+  --region_mask REGION_MASK_FNAME
+                POP region mask file (to specify open ocean vs marginal sea)
+  --map_out MAP_OUT_FNAME
+                output file
+EOF
+}
+
+################################################################################
+
+parse_args() {
+
+  # Clear pre-existing environment variables that will be set
+  export MAP_IN_OO_FNAME=""
+  export MAP_IN_MS_FNAME=""
+  export REGION_MASK_FNAME=""
+  export MAP_OUT_FNAME=""
+
+  while [ $# -gt 0 ]; do
+    case $1 in
+      --map_in_oo )
+        export MAP_IN_OO_FNAME=$2
+        shift
+      ;;
+      --map_in_ms )
+        export MAP_IN_MS_FNAME=$2
+        shift
+      ;;
+      --region_mask )
+        export REGION_MASK_FNAME=$2
+        shift
+      ;;
+      --map_out )
+        export MAP_OUT_FNAME=$2
+        shift
+      ;;
+      -v | --verbose )
+        VERBOSE=TRUE
+      ;;
+      -h | --help )
+        usage
+        exit 0
+      ;;
+      * )
+        echo "ERROR: $1 is not a valid argument"
+        echo ""
+        echo "Run $0 -h for help"
+        exit 1
+    esac
+    shift
+  done
+}
+
+################################################################################
+
+check_inputs() {
+
+  ERROR=""
+
+  if [ -z ${MAP_IN_OO_FNAME} ]; then
+    echo "ERROR: Missing --map_in_oo option"
+    ERROR="TRUE"
+  fi
+
+  if [ -z ${MAP_IN_MS_FNAME} ]; then
+    echo "ERROR: Missing --map_in_ms option"
+    ERROR="TRUE"
+  fi
+
+  if [ -z ${REGION_MASK_FNAME} ]; then
+    echo "ERROR: Missing --region_mask option"
+    ERROR="TRUE"
+  fi
+
+  if [ -z ${MAP_OUT_FNAME} ]; then
+    echo "ERROR: Missing --map_out option"
+    ERROR="TRUE"
+  fi
+
+  if [ ! -z ${ERROR} ]; then
+    echo ""
+    echo "Run $0 -h for help"
+    exit 1
+  fi
+}
+
+################################################################################
+
+print_summary() {
+  echo "Open ocean map: $MAP_IN_OO_FNAME"
+  echo "Marginal seas map: $MAP_IN_MS_FNAME"
+  echo "Region mask: $REGION_MASK_FNAME"
+  echo "Output map: $MAP_OUT_FNAME"
+}
+
+################################################################################
+
+main $@
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile
index fa46862ab1d..baf1f6cf754 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile
@@ -88,6 +88,7 @@ INCLDIR  += -I$(NETCDF_PATH)/include
 # echo file names, paths, compile flags, etc. used during build
 #-------------------------------------------------------------------------------
 
+.PHONY: db_files
 db_files:
 	@echo " "
 	@echo "* EXEC    := $(EXEC)"
@@ -99,6 +100,8 @@ db_files:
 	@echo "* SRCS    := $(SRCS)"
 	@echo "* OBJS    := $(OBJS)"
 	@echo "* DEPS    := $(DEPS)"
+
+.PHONY: db_flags
 db_flags:
 	@echo " "
 	@echo "* cc      := $(SCC)  $(CFLAGS) $(INCS) $(INCLDIR)"
@@ -129,14 +132,13 @@ else
 	$(SFC)  $(FFLAGS) $(FREEFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $*.F90
 endif
 
+.PHONY: clean
 clean:
-	$(RM) -f *.f *.f90 *.d *.mod $(OBJS)
-
-realclean: clean
-	$(RM) -f $(EXEC)
+	$(RM) -f *.f *.f90 *.d *.mod $(OBJS) $(EXEC) $(DEPGEN)
 
-distclean: realclean
-	$(RM) -f $(MACFILE) env_mach_specific Depends* makdep build.csh
+.PHONY: distclean
+distclean: clean
+	../../../../configure --clean
 
 #-------------------------------------------------------------------------------
 # Build & include dependency files
@@ -169,12 +171,10 @@ $(DEPGEN) :
 # the if-tests prevent DEPS files from being created when they're not needed
 ifneq ($(MAKECMDGOALS), db_files)
 ifneq ($(MAKECMDGOALS), db_flags)
-ifneq ($(MAKECMDGOALS), mostlyclean)
 ifneq ($(MAKECMDGOALS), clean)
-ifneq ($(MAKECMDGOALS), realclean)
+ifneq ($(MAKECMDGOALS), distclean)
     -include $(DEPS)
 endif
 endif
 endif
 endif
-endif
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90
index 89cf3e49981..fbcab4d30cc 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: fixroff_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90 $
-!===============================================================================
-
 MODULE fixroff_mod
 
    use shr_sys_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90
index 6fb4065ff1c..515219ed7cf 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: kind_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90 $
-!===============================================================================
-
 MODULE kind_mod
 
    use shr_kind_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90
index f109eb3fff7..a415e633ee4 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: main.F90 59443 2014-04-22 22:57:10Z mlevy@ucar.edu $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/main.F90 $
-!===============================================================================
-
 PROGRAM main
 
    use shr_timer_mod
@@ -32,31 +27,36 @@ PROGRAM main
    character(10)     :: tstr            ! wall clock time
 
    !--- namelist vars ---
-   character(180) :: gridtype      ! type of run-off grid
-   character(180) :: file_roff     ! file name: rtm rdirc file
-   character(180) :: file_roff_out ! file name: rtm rdirc file
-   character(180) :: file_ocn      ! file name: ocn scrip grid file
-   character(180) :: file_nn       ! file name: orig matrix, corrected
-   character(180) :: file_new      ! file name: orig matrix, corrected, smoothed, sorted -- done
-   character(180) :: file_smooth   ! file name: smoothing matrix
-   character(180) :: title         ! netCDF title attribute
-   logical        :: step1         ! gen nn
-   logical        :: step2         ! gen smooth
-   logical        :: step3         ! mat mult
-   logical        :: lmake_rSCRIP  ! .true. => convert runoff grid to SCRIP
-
-   namelist / input_nml / &
-      gridtype      &
-   ,  file_roff     &
-   ,  file_roff_out &
-   ,  file_ocn      &
-   ,  file_nn       &
-   ,  file_new      &
-   ,  file_smooth   &
-   ,  title         &
-   ,  eFold         &
-   ,  rMax          &
-   ,  lmake_rSCRIP  &
+   character(180) :: gridtype                        ! type of run-off grid
+   character(180) :: file_roff                       ! file name: rtm rdirc file
+   character(180) :: file_roff_out                   ! file name: rtm rdirc file
+   character(180) :: file_ocn                        ! file name: ocn scrip grid file (all ocean points masked as such)
+   character(180) :: file_ocn_coastal_mask           ! file name: ocn scrip grid file (only coastal points masked as ocean)
+   character(180) :: file_nn                         ! file name: orig matrix, corrected
+   character(180) :: file_new                        ! file name: orig matrix, corrected, smoothed, sorted -- done
+   character(180) :: file_smooth                     ! file name: smoothing matrix
+   character(180) :: title                           ! netCDF title attribute
+   logical        :: restrict_smooth_src_to_nn_dest  ! use step1 dest cells as
+                                                     ! step2 source cells
+   logical        :: step1                           ! gen nn
+   logical        :: step2                           ! gen smooth
+   logical        :: step3                           ! mat mult
+   logical        :: lmake_rSCRIP                    ! .true. => convert runoff grid to SCRIP
+
+   namelist / input_nml /            &
+      gridtype                       &
+   ,  file_roff                      &
+   ,  file_roff_out                  &
+   ,  file_ocn                       &
+   ,  file_ocn_coastal_mask          &
+   ,  file_nn                        &
+   ,  file_new                       &
+   ,  file_smooth                    &
+   ,  title                          &
+   ,  eFold                          &
+   ,  rMax                           &
+   ,  lmake_rSCRIP                   &
+   ,  restrict_smooth_src_to_nn_dest &
    ,  step1, step2, step3
 
    !--- formats ---
@@ -80,8 +80,6 @@ PROGRAM main
 !-------------------------------------------------------------------------------
 
    write(6,F10) "correct/smooth/sort runoff -> ocean map"
-   write(6,F10) "SVN &
-   & $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/main.F90 $"
 
    call shr_timer_init()
 
@@ -89,39 +87,45 @@ PROGRAM main
    write(6,F10) "Step 0:  read input namelist data"
    !----------------------------------------------------------------------------
 
-   gridtype      = 'unset'
-   file_roff     = 'unset'
-   file_ocn      = 'unset'
-   file_nn       = 'unset'
-   file_smooth   = 'unset'
-   file_new      = 'unset'
-   title         = 'unset'
-   eFold         = 1000000.00000 ! smoothing eFold distance in meters
-   rMax          =  500000.00000 ! max smoothing radius in meters
-   step1         = .true.
-   step2         = .true.
-   step3         = .true.
+   gridtype                       = 'unset'
+   file_roff                      = 'unset'
+   file_ocn                       = 'unset'
+   file_ocn_coastal_mask          = 'unset'
+   file_nn                        = 'nn.nc'
+   file_smooth                    = 'smooth.nc'
+   file_new                       = 'nnsm.nc'
+   title                          = 'unset'
+   restrict_smooth_src_to_nn_dest = .true.
+   eFold                          = 1000000._R8 ! smoothing eFold distance in meters
+   rMax                           =  300000._R8 ! max smoothing radius in meters
+   step1                          = .true.
+   step2                          = .true.
+   step3                          = .true.
 
    ! These two variables typically don't appear in namelist
    lmake_rSCRIP  = .false.
    file_roff_out = "runoff.nc"
 
-   open (10,file="runoff_map.nml",status="old",action="read")
-   read (10,nml=input_nml,iostat=rCode)
-   close(10)
+   read (*,nml=input_nml,iostat=rCode)
+   if (trim(file_ocn_coastal_mask) == 'unset') then
+     file_ocn_coastal_mask = file_ocn
+   end if
+
    write(6,F00) "Namelist values..."
-   write(6,F00) "   gridtype       = ",trim(gridtype      )
-   write(6,F00) "   file_roff      = ",trim(file_roff     )
-   write(6,F00) "   file_ocn       = ",trim(file_ocn      )
-   write(6,F00) "   file_nn        = ",trim(file_nn       )
-   write(6,F00) "   file_smooth    = ",trim(file_smooth   )
-   write(6,F00) "   file_new       = ",trim(file_new      )
-   write(6,F00) "   title          = ",trim(title         )
-   write(6,F02) "   eFold distance = ",eFold
-   write(6,F02) "   rMax  distance = ",rMax
-   write(6,F03) "   step1          = ",step1
-   write(6,F03) "   step2          = ",step2
-   write(6,F03) "   step3          = ",step3
+   write(6,F00) "   gridtype                       = ",trim(gridtype      )
+   write(6,F00) "   file_roff                      = ",trim(file_roff     )
+   write(6,F00) "   file_ocn (global)              = ",trim(file_ocn)
+   write(6,F00) "   file_ocn (coast)               = ",trim(file_ocn_coastal_mask)
+   write(6,F00) "   file_nn                        = ",trim(file_nn       )
+   write(6,F00) "   file_smooth                    = ",trim(file_smooth   )
+   write(6,F00) "   file_new                       = ",trim(file_new      )
+   write(6,F00) "   title                          = ",trim(title         )
+   write(6,F02) "   eFold distance                 = ",eFold
+   write(6,F02) "   rMax  distance                 = ",rMax
+   write(6,F03) "   restrict_smooth_src_to_nn_dest = ",restrict_smooth_src_to_nn_dest
+   write(6,F03) "   step1                          = ",step1
+   write(6,F03) "   step2                          = ",step2
+   write(6,F03) "   step3                          = ",step3
 !  if (rCode > 0) then
 !     write(6,F01) 'ERROR: reading input namelist, iostat=',rCode
 !     stop
@@ -147,7 +151,7 @@ PROGRAM main
    !----------------------------------------------------------------------------
    write(6,F10) "Step 1: read grid info & create nearest neighbor map"
    !----------------------------------------------------------------------------
-   call map_gridRead(map_orig , trim(file_roff), trim(file_ocn), gridtype)
+   call map_gridRead(map_orig , trim(file_roff), trim(file_ocn_coastal_mask), gridtype)
 
    call date_and_time(dstr,tstr)
    call map_print(map_orig)
@@ -176,7 +180,7 @@ PROGRAM main
    !----------------------------------------------------------------------------
    write(6,F02) "   eFold distance = ",eFold
    write(6,F02) "   rMax  distance = ",rMax
-   call smooth_init(map_orig,map_smooth)
+   call smooth_init(file_ocn, restrict_smooth_src_to_nn_dest, map_orig,map_smooth)
 !  call restrictSources(map_smooth,trim(file_sources)) ! restrict cells subject to smoothing
    call smooth(map_smooth,eFold,rMax)
    call mapsort_sort(map_smooth)
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90
index 38e637bbbef..7cf87a7d846 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: map_mod.F90 56089 2013-12-18 00:50:07Z mlevy@ucar.edu $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/map_mod.F90 $
-!===============================================================================
-
 MODULE map_mod
 
    use shr_sys_mod
@@ -385,7 +380,6 @@ SUBROUTINE map_gridRead(map, rfilename, ofilename, gridtype, lmake_rSCRIP)
    map%convention = 'NCAR-CCSM'
    map%history    = 'history'
    map%domain_a   = trim(rfilename)
-   map%domain_b   = trim(ofilename)
 
    if (trim(gridtype) == "rtm") then
       !-------------------------------------------------------------------------
@@ -746,88 +740,7 @@ SUBROUTINE map_gridRead(map, rfilename, ofilename, gridtype, lmake_rSCRIP)
    !----------------------------------------------------------------------------
    write(*,F00) "read destination domain info -- pop grid"
    !----------------------------------------------------------------------------
-   write(6,F00) 'ocn data file',' = ',trim(ofilename)
-   rcode = nf_open(ofilename,NF_NOWRITE,fid)
-
-   rcode = nf_inq_dimid (fid, 'grid_size' , did)
-   rcode = nf_inq_dimlen(fid, did   , map%n_b  )
-   rcode = nf_inq_dimid (fid, 'grid_corners' , did)
-   rcode = nf_inq_dimlen(fid, did   , map%nv_b  )
-   rcode = nf_inq_dimid (fid, 'grid_rank', did)
-   rcode = nf_inq_dimlen(fid, did   , grid_rank)
-   allocate(grid_dims(grid_rank))
-   rcode = nf_inq_varid  (fid, 'grid_dims', vid)
-   rcode = nf_get_var_int(fid, vid   , grid_dims)
-   if (grid_rank.eq.1) then
-     map%ni_b = grid_dims(1)
-     map%nj_b = 1
-   elseif (grid_rank.eq.2) then
-     map%ni_b = grid_dims(1)
-     map%nj_b = grid_dims(2)
-   else
-      deallocate(grid_dims)
-      write(6,*) 'ERROR: grid_rank is ',grid_rank,' in ',trim(ofilename)
-      call shr_sys_abort(subName//"ERROR: ofilename grid_rank")
-   endif
-   deallocate(grid_dims)
-   map%dims_b(1) = map%ni_b
-   map%dims_b(2) = map%nj_b
-
-   allocate(map%  xc_b(         map%n_b)) ! x-coordinates of center
-   allocate(map%  yc_b(         map%n_b)) ! y-coordinates of center
-   allocate(map%  xv_b(map%nv_b,map%n_b)) ! x-coordinates of verticies
-   allocate(map%  yv_b(map%nv_b,map%n_b)) ! y-coordinates of verticies
-   allocate(map%mask_b(         map%n_b)) ! domain mask
-   allocate(map%area_b(         map%n_b)) ! grid cell area
-   allocate(map%frac_b(         map%n_b)) ! grid cell area
-   allocate(map%sn1            (map%n_b))
-   allocate(map%sn2            (map%n_b))
-
-   rcode = nf_inq_varid     (fid,'grid_center_lon'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%xc_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%xc_b = map%xc_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_center_lat'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%yc_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%yc_b = map%yc_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_corner_lon'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%xv_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%xv_b = map%xv_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_corner_lat'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%yv_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%yv_b = map%yv_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_imask',vid )
-   rcode = nf_get_var_int   (fid,vid     ,map%mask_b)
-   rcode = nf_inq_varid     (fid,'grid_area',vid )
-   if (rcode.eq.0) then
-      rcode = nf_get_var_double(fid,vid     ,map%area_b)
-   else
-      write(6,*) "ERROR: could not find variable grid_area in destination grid input file!"
-      stop
-   end if
-
-   map%frac_b = map%mask_b * 1.0_r8
-
-   rcode = nf_close(fid)
+   call map_DestGridRead(map, ofilename)
 
    !----------------------------------------------------------------------------
    write(*,F00) "derive info required to compute NN map"
@@ -1474,10 +1387,6 @@ SUBROUTINE map_write(map, filename)
    rcode = nf_put_att_text(fid,NF_GLOBAL,'normalization',len_trim(str),str)
     str  = map%method
    rcode = nf_put_att_text(fid,NF_GLOBAL,'map_method' ,len_trim(str),str)
-    str  = "$SVN"
-   rcode = nf_put_att_text(fid,NF_GLOBAL,'SVN URL'     ,len_trim(str),str)
-    str  = "$Id"
-   rcode = nf_put_att_text(fid,NF_GLOBAL,'SVN Id'      ,len_trim(str),str)
     str  = map%history
    call date_and_time(cdate,ctime) ! f90 intrinsic
     str = 'File created: '//cdate(1:4)//'-'//cdate(5:6)//'-'//cdate(7:8) &
@@ -2593,6 +2502,119 @@ real(r8) FUNCTION map_distance(x0,y0,x1,y1)
 
 END FUNCTION map_distance
 
+!===============================================================================
+
+SUBROUTINE map_DestGridRead(map, filename)
+
+   !--- modules ---
+
+   implicit none
+
+   !--- includes ---
+
+   !--- arguments ---
+   type(sMatrix), intent(inout) :: map       ! sMatrix info to be read in
+   character(*) , intent(in)    :: filename  ! name of data file
+
+   !--- local ---
+   character(strLen)     :: units     ! netCDF attribute name string
+   integer               :: rcode   ! netCDF routine return code
+   integer               :: fid     ! netCDF file      ID
+   integer               :: vid     ! netCDF variable  ID
+   integer               :: did     ! netCDF dimension ID
+   integer               :: grid_rank
+   integer, allocatable, dimension(:) :: grid_dims
+
+   character(*),parameter :: subName = "(map_DestGridRead) "
+   !--- formats ---
+   character(len=*),parameter :: F00 = "('(map_DestGridRead) ',3a)"
+   character(len=*),parameter :: F02 = "('(map_DestGridRead) ',a11,a3,60(a1))"
+
+   write(6,F00) 'ocn data file',' = ',trim(filename)
+   rcode = nf_open(filename,NF_NOWRITE,fid)
+
+   rcode = nf_inq_dimid (fid, 'grid_size' , did)
+   rcode = nf_inq_dimlen(fid, did   , map%n_b  )
+   rcode = nf_inq_dimid (fid, 'grid_corners' , did)
+   rcode = nf_inq_dimlen(fid, did   , map%nv_b  )
+   rcode = nf_inq_dimid (fid, 'grid_rank', did)
+   rcode = nf_inq_dimlen(fid, did   , grid_rank)
+   allocate(grid_dims(grid_rank))
+   rcode = nf_inq_varid  (fid, 'grid_dims', vid)
+   rcode = nf_get_var_int(fid, vid   , grid_dims)
+   if (grid_rank.eq.1) then
+     map%ni_b = grid_dims(1)
+     map%nj_b = 1
+   elseif (grid_rank.eq.2) then
+     map%ni_b = grid_dims(1)
+     map%nj_b = grid_dims(2)
+   else
+      deallocate(grid_dims)
+      write(6,*) 'ERROR: grid_rank is ',grid_rank,' in ',trim(filename)
+      call shr_sys_abort(subName//"ERROR: filename grid_rank")
+   endif
+   deallocate(grid_dims)
+   map%dims_b(1) = map%ni_b
+   map%dims_b(2) = map%nj_b
+
+   allocate(map%  xc_b(         map%n_b)) ! x-coordinates of center
+   allocate(map%  yc_b(         map%n_b)) ! y-coordinates of center
+   allocate(map%  xv_b(map%nv_b,map%n_b)) ! x-coordinates of verticies
+   allocate(map%  yv_b(map%nv_b,map%n_b)) ! y-coordinates of verticies
+   allocate(map%mask_b(         map%n_b)) ! domain mask
+   allocate(map%area_b(         map%n_b)) ! grid cell area
+   allocate(map%frac_b(         map%n_b)) ! grid cell area
+   allocate(map%sn1            (map%n_b))
+   allocate(map%sn2            (map%n_b))
+
+   rcode = nf_inq_varid     (fid,'grid_center_lon'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%xc_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%xc_b = map%xc_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_center_lat'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%yc_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%yc_b = map%yc_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_corner_lon'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%xv_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%xv_b = map%xv_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_corner_lat'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%yv_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%yv_b = map%yv_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_imask',vid )
+   rcode = nf_get_var_int   (fid,vid     ,map%mask_b)
+   rcode = nf_inq_varid     (fid,'grid_area',vid )
+   if (rcode.eq.0) then
+      rcode = nf_get_var_double(fid,vid     ,map%area_b)
+   else
+      write(6,*) "ERROR: could not find variable grid_area in destination grid input file!"
+      stop
+   end if
+
+   map%frac_b = map%mask_b * 1.0_r8
+   map%domain_b = trim(filename)
+   rcode = nf_close(fid)
+
+END SUBROUTINE map_DestGridRead
+
 !===============================================================================
 !===============================================================================
 END MODULE map_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90
index 250c8865a5a..df882af8902 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: mapsort_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90 $
-!===============================================================================
-
 MODULE mapsort_mod
 
    use shr_sys_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90
index 2a2dd64512a..de361d8d747 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: shr_kind_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90 $
-!===============================================================================
-
 MODULE shr_kind_mod
 
    !----------------------------------------------------------------------------
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90
index b7aba1066e2..0cd2f896937 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: shr_sys_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90 $
-!===============================================================================
-
 MODULE shr_sys_mod
 
    use shr_kind_mod  ! defines real & integer kinds
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90
index ffa9deebaf1..3acf86b8976 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: shr_timer_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90 $
-!===============================================================================
-
 module shr_timer_mod
 
    !----------------------------------------------------------------------------
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90
index 03f889db6d8..b7cc1966b61 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: smooth_mod.F90 56089 2013-12-18 00:50:07Z mlevy@ucar.edu $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90 $
-!===============================================================================
-
 MODULE smooth_mod
 
 #define _NEW 1
@@ -26,13 +21,26 @@ MODULE smooth_mod
 CONTAINS
 !===============================================================================
 
-SUBROUTINE smooth_init(map_in, map_out)
+SUBROUTINE smooth_init(ofilename, restrict_smooth_src_to_nn_dest,  map_in, map_out)
 
    implicit none
 
    !--- arguments ---
-   type(sMatrix),intent( in)   :: map_in   ! original unsmoothed, matrix
-   type(sMatrix),intent(inout) :: map_out  ! smoothing matrix
+   ! name of ocn scrip grid file
+   character(*), intent(in)    :: ofilename
+
+   ! original unsmoothed, matrix
+   type(sMatrix),intent(in)    :: map_in
+
+   ! map_out%mask_a = pts mapped to map_in?
+   ! This should be set to .true. if you are making a single map, but it may be
+   ! useful to set it to .false. if you plan on mapping several runoff grids to
+   ! the same ocean grid. The nearest-neighbor / smooth map generated in step 3
+   ! will be the same regardless of the value of this variable.
+   logical,      intent(in)    :: restrict_smooth_src_to_nn_dest
+
+   ! smoothing matrix
+   type(sMatrix),intent(inout) :: map_out
 
    !--- local ---
    integer :: i,j,n ! indicies: row, col, sparse matrix
@@ -123,26 +131,19 @@ SUBROUTINE smooth_init(map_in, map_out)
    allocate(map_out%  yv_a(map_in%nv_b,map_in%n_b) )
    allocate(map_out%mask_a(            map_in%n_b) )
    allocate(map_out%area_a(            map_in%n_b) )
-
-   allocate(map_out%  xc_b(            map_in%n_b) )
-   allocate(map_out%  yc_b(            map_in%n_b) )
-   allocate(map_out%  xv_b(map_in%nv_b,map_in%n_b) )
-   allocate(map_out%  yv_b(map_in%nv_b,map_in%n_b) )
-   allocate(map_out%mask_b(            map_in%n_b) )
-   allocate(map_out%area_b(            map_in%n_b) )
-
    allocate(map_out%frac_a(map_in%n_b) )
-   allocate(map_out%frac_b(map_in%n_b) )
 
    allocate(map_out%s  (ns))
    allocate(map_out%row(ns))
    allocate(map_out%col(ns))
-   allocate(map_out%sn1(map_in%n_b) )
-   allocate(map_out%sn2(map_in%n_b) )
 
    !------------------------------------------------
    ! set values
    !------------------------------------------------
+
+   ! map_in maps from runoff -> coastal ocean
+   ! map_out maps from coastal ocean -> global ocean
+   ! So source of map_out = dest of map_in
    map_out%   n_a = map_in%   n_b
    map_out%dims_a = map_in%dims_b
    map_out%  ni_a = map_in%  ni_b
@@ -152,36 +153,30 @@ SUBROUTINE smooth_init(map_in, map_out)
    map_out%  yc_a = map_in%  yc_b
    map_out%  xv_a = map_in%  xv_b
    map_out%  yv_a = map_in%  yv_b
-!  map_out%mask_a = map_in%mask_b ! all active ocn cells
    map_out%area_a = map_in%area_b
+   map_out%domain_a   = map_in%domain_b
 
-   !--- compute minimal src domain mask for smoothing matrix ---
-   jmd_count = 0
-   map_out%mask_a = 0
-   do n=1,map_in%n_s
-      i = map_in%row(n)     ! this ocn cell could get runoff
-      map_out%mask_a(i) = 1 ! this ocn cell's runoff get's smoothed
-      if(map_in%s(n) > 0.0) then
-        jmd_count = jmd_count+1
-      endif
-   end do
-   write(*,*) subName,'number of source points is =  ',jmd_count
-   write(*,*) subName,'map_in%ns                  =  ',map_in%n_s
-
-   map_out%   n_b = map_in%   n_b
-   map_out%dims_b = map_in%dims_b
-   map_out%  ni_b = map_in%  ni_b
-   map_out%  nj_b = map_in%  nj_b
-   map_out%  nv_b = map_in%  nv_b
-   map_out%  xc_b = map_in%  xc_b
-   map_out%  yc_b = map_in%  yc_b
-   map_out%  xv_b = map_in%  xv_b
-   map_out%  yv_b = map_in%  yv_b
-   map_out%mask_b = map_in%mask_b
-   map_out%area_b = map_in%area_b
-
-!  map_out%frac_a = map_in%frac_b
-!  map_out%frac_b = map_in%frac_b
+   if (restrict_smooth_src_to_nn_dest) then
+      !--- compute minimal src domain mask for smoothing matrix ---
+      jmd_count = 0
+      map_out%mask_a = 0
+      do n=1,map_in%n_s
+         i = map_in%row(n)     ! this ocn cell could get runoff
+         map_out%mask_a(i) = 1 ! this ocn cell's runoff get's smoothed
+         if(map_in%s(n) > 0.0) then
+            jmd_count = jmd_count+1
+         endif
+      end do
+      write(*,*) subName,'number of source points is =  ',jmd_count
+      write(*,*) subName,'map_in%ns                  =  ',map_in%n_s
+   else
+      map_out%mask_a = map_in%mask_b ! all active ocn cells
+   end if
+
+   ! destination of map_out must be read in from file
+   call map_DestGridRead(map_out, ofilename)
+
+   ! Overwrite frac_a and frac_b to be 1 globally
    map_out%frac_a = 1.0
    map_out%frac_b = 1.0
 
@@ -189,16 +184,11 @@ SUBROUTINE smooth_init(map_in, map_out)
    map_out%s      = 1.0
    map_out%row    = 1
    map_out%col    = 1
-   map_out%sn1    = map_in%sn1
-   map_out%sn2    = map_in%sn2
 
    map_out%title      = "CCSM conservative smoothing map"
    map_out%normal     = map_in%normal
-   map_out%method     = "created using SVN $Id: smooth_mod.F90 56089 2013-12-18 00:50:07Z mlevy@ucar.edu $"
    map_out%history    = map_in%history
    map_out%convention = map_in%convention
-   map_out%domain_a   = map_in%domain_b
-   map_out%domain_b   = map_in%domain_b
 
 END SUBROUTINE smooth_init
 
@@ -394,9 +384,9 @@ SUBROUTINE smooth(map,efold,rmax)
         wgtsum = 0.0
         nbox = 0
         gdcnt = 0
+        kStart = 1
 
 #ifdef _BREADTH
-        kStart = 2
         i2ind(1) = ic
         j2ind(1) = jc
         imask_JMD = -1000
@@ -408,7 +398,6 @@ SUBROUTINE smooth(map,efold,rmax)
                 rdist,rmax,i2ind,j2ind, strPtr, length)
 
 #else
-        kStart = 1
         !--- recursive function to find dest cells ---
 
         call depth_setDist(ic,jc,ni,nj,map%xc_a,map%yc_a,0.0,imask,rdist,rmax,i2ind,j2ind,length)
diff --git a/tools/mapping/map_field/src/map_field.F90 b/tools/mapping/map_field/src/map_field.F90
index a612772d359..3f0a9c132bc 100644
--- a/tools/mapping/map_field/src/map_field.F90
+++ b/tools/mapping/map_field/src/map_field.F90
@@ -469,7 +469,8 @@ subroutine write_file(fid, fout, units, n, ni, nj, &
        if (rcode == 0) then
           call check_ret(nf_put_att_text(fid,NF_GLOBAL,'hostname' ,len_trim(host),host))
        else
-          write(6,*) 'WARNING: could not determine hostname, so that information will not be stored in netCDF attribute. To avoid this warning in the future, set environment variable HOST or HOSTNAME.'
+          write(6,*) 'WARNING: could not determine hostname, so that information will not be &
+               &stored in netCDF attribute. To avoid this warning in the future, set environment variable HOST or HOSTNAME.'
        end if
     end if
 
diff --git a/tools/unit_testing/run_tests.py b/tools/unit_testing/run_tests.py
index 34359a4f8e5..e7a152fe200 100755
--- a/tools/unit_testing/run_tests.py
+++ b/tools/unit_testing/run_tests.py
@@ -64,9 +64,9 @@ def parse_command_line(args):
     parser.add_argument("--no-color",  action="store_false",
                       help="""Turn off colorized output."""
     )
-    parser.add_argument("--compiler",  default="gnu",
+    parser.add_argument("--compiler",
                         help="""Compiler vendor for build (supported depends on machine).
-Only used for lookup in CIME Machines files."""
+                        If not specified, use the default for this machine."""
                         )
     parser.add_argument("--enable-genf90", action="store_true",
                         default=True,
@@ -83,19 +83,23 @@ def parse_command_line(args):
 
     parser.add_argument("--mpilib",
     help="""MPI Library to use in build.
-Required argument (until we can get this from config_machines)
-Must match an MPILIB option in config_compilers.xml.
-e.g., for yellowstone, can use 'mpich2'."""
+    If not specified, use the default for this machine/compiler.
+    Must match an MPILIB option in config_compilers.xml.
+    e.g., for yellowstone, can use 'mpich2'."""
     )
-    parser.add_argument(
-        "--mpirun-command", default="",
-        help="""Command to use to run an MPI executable.
 
-If not specified, does not use any mpirun prefix to run executables."""
+    parser.add_argument("--no-mpirun", action="store_true",
+                        help="""If specified, then run executables without an mpirun command""")
+
+    parser.add_argument("--mpirun-command",
+        help="""Command to use to run an MPI executable.
+        If not specified, uses the default for this machine.
+        Ignored if --no-mpirun is specified."""
         )
     parser.add_argument(
-        "--test-spec-dir",
-        help="""Location where tests are specified."""
+        "--test-spec-dir", default=".",
+        help="""Location where tests are specified.
+        Defaults to current directory."""
         )
     parser.add_argument(
     "-T", "--ctest-args",
@@ -110,9 +114,8 @@ def parse_command_line(args):
 override the command provided by Machines."""
     )
     parser.add_argument(
-        "--use-openmp",  action="store_true",
-        default=False,
-        help="""Turn on OPENMP support for tests."""
+        "--no-openmp",  action="store_true",
+        help="""Default is to include OPENMP support for tests; this option excludes openmp"""
     )
     parser.add_argument(
         "--xml-test-list",
@@ -133,8 +136,8 @@ def parse_command_line(args):
 
     return output, args.build_dir, args.build_optimized, args.clean,\
         args.cmake_args, args.compiler, args.enable_genf90, args.machine, args.machines_dir,\
-        args.make_j, args.mpilib, args.mpirun_command, args.test_spec_dir, args.ctest_args,\
-        args.use_openmp, args.xml_test_list, args.verbose
+        args.make_j, args.mpilib, args.no_mpirun, args.mpirun_command, args.test_spec_dir, args.ctest_args,\
+        args.no_openmp, args.xml_test_list, args.verbose
 
 
 def cmake_stage(name, test_spec_dir, build_optimized, mpirun_command, output, cmake_args=None, clean=False, verbose=False, enable_genf90=True, color=True):
@@ -167,9 +170,9 @@ def cmake_stage(name, test_spec_dir, build_optimized, mpirun_command, output, cm
         cmake_command = [
             "cmake",
             test_spec_dir,
-            "-DCIME_CMAKE_MODULE_DIRECTORY="+os.path.abspath(os.path.join(_CIMEROOT,"externals","CMake")),
+            "-DCIME_CMAKE_MODULE_DIRECTORY="+os.path.abspath(os.path.join(_CIMEROOT,"src","externals","CMake")),
             "-DCMAKE_BUILD_TYPE="+build_type,
-            "-DPFUNIT_MPIRUN="+mpirun_command,
+            "-DPFUNIT_MPIRUN='"+mpirun_command+"'",
             ]
         if verbose:
             cmake_command.append("-Wdev")
@@ -177,7 +180,7 @@ def cmake_stage(name, test_spec_dir, build_optimized, mpirun_command, output, cm
         if enable_genf90:
             cmake_command.append("-DENABLE_GENF90=ON")
             genf90_dir = os.path.join(
-                _CIMEROOT, "externals", "genf90"
+                _CIMEROOT,"src","externals","genf90"
                 )
             cmake_command.append("-DCMAKE_PROGRAM_PATH="+genf90_dir)
 
@@ -216,10 +219,12 @@ def make_stage(name, output, make_j, clean=False, verbose=True):
 def _main():
     output, build_dir, build_optimized, clean,\
         cmake_args, compiler, enable_genf90, machine, machines_dir,\
-        make_j, mpilib, mpirun_command, test_spec_dir, ctest_args,\
-        use_openmp, xml_test_list, verbose \
+        make_j, mpilib, no_mpirun, mpirun_command, test_spec_dir, ctest_args,\
+        no_openmp, xml_test_list, verbose \
         = parse_command_line(sys.argv)
 
+    use_openmp = not no_openmp
+
 #=================================================
 # Find directory and file paths.
 #=================================================
@@ -241,14 +246,6 @@ def _main():
             )
 
 
-
-
-# Search for the CESM root directory.
-# First check the option. If not specified, look to see if there's a tools
-# directory two levels up (just as a sanity check).
-    if machine is None:
-        machine="yellowstone"
-
     if machines_dir is not None:
         machines_file = os.path.join(machines_dir, "config_machines.xml")
         machobj = Machines(infile=machines_file, machine=machine)
@@ -269,8 +266,10 @@ def _main():
 #=================================================
     if mpilib is None:
         mpilib = machobj.get_default_MPIlib()
+        logger.warn("Using mpilib: %s"%mpilib)
     if compiler is None:
         compiler = machobj.get_default_compiler()
+        logger.warn("Compiler is %s"%compiler)
 
     debug = not build_optimized
     os_ = machobj.get_value("OS")
@@ -280,8 +279,22 @@ def _main():
     #
     configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_)
     machspecific = EnvMachSpecific(build_dir)
+
+    if no_mpirun:
+        mpirun_command = ""
+    elif mpirun_command is None:
+        mpi_attribs = {
+            "compiler" : compiler,
+            "mpilib"   : mpilib,
+            "threaded" : use_openmp,
+            "unit_testing" : True
+        }
+
+        # We can get away with specifying case=None since we're using exe_only=True
+        mpirun_command, _ = machspecific.get_mpirun(case=None, attribs=mpi_attribs, exe_only=True)
+        logger.warn("mpirun command is '%s'"%mpirun_command)
+
     machspecific.load_env(compiler, debug, mpilib)
-    logger.warn("Compiler is %s"%compiler)
     os.environ["OS"] = os_
     os.environ["COMPILER"] = compiler
     os.environ["DEBUG"] = stringify_bool(debug)
diff --git a/utils/perl5lib/compilers_translation_tool.pl b/utils/perl5lib/compilers_translation_tool.pl
index 964c734d3bb..15ed6ba02fa 100755
--- a/utils/perl5lib/compilers_translation_tool.pl
+++ b/utils/perl5lib/compilers_translation_tool.pl
@@ -17,7 +17,7 @@
     confirmed by running:
 
     xmllint -noout -schema \
-        \$CIME_ROOT/cime_config/xml_schemas/config_build.xsd OUTPUT_FILE
+        \$CIME_ROOT/config/xml_schemas/config_build.xsd OUTPUT_FILE
 EOF
 }