From 603d2f72db1a929d9eb5ac0d4d08582a7cdc9586 Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 13 Jun 2018 13:46:11 -0500 Subject: [PATCH 1/2] Adding tests for extra dimensions in the decomposition In PIO1, a decomposition can have extra outermost dimensions with their lengths all set to 1. For example, we can define a 3D decomposition of [1 x 10 x 20] to read/write a 2D variable of [10 x 20], or a 3D record variable of [time x 10 x 20]. We need to test non-record variable, record variable (uses unlimited time dimension) and quasi-record variable (uses limited time dimension) for extra dimensions. For these tests to pass, existing PIO2 code needs to be updated to get matched start/count values from the decomposition. --- tests/general/CMakeLists.txt | 20 + tests/general/pio_decomp_extra_dims.F90.in | 896 +++++++++++++++++++++ 2 files changed, 916 insertions(+) create mode 100644 tests/general/pio_decomp_extra_dims.F90.in diff --git a/tests/general/CMakeLists.txt b/tests/general/CMakeLists.txt index 54e6c62b91..1aaca7246a 100644 --- a/tests/general/CMakeLists.txt +++ b/tests/general/CMakeLists.txt @@ -30,6 +30,7 @@ SET(GENERATED_SRCS pio_file_simple_tests.F90 pio_decomp_frame_tests.F90 pio_decomp_fillval.F90 pio_decomp_fillval2.F90 + pio_decomp_extra_dims.F90 pio_iosystem_tests.F90 pio_iosystem_tests2.F90 pio_iosystem_tests3.F90) @@ -832,6 +833,25 @@ else () TIMEOUT ${DEFAULT_TEST_TIMEOUT}) endif () +#===== pio_decomp_extra_dims ===== +add_executable (pio_decomp_extra_dims EXCLUDE_FROM_ALL + pio_decomp_extra_dims.F90 + ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) +target_link_libraries (pio_decomp_extra_dims piof) +add_dependencies (tests pio_decomp_extra_dims) + +if (PIO_USE_MPISERIAL) + add_test(NAME pio_decomp_extra_dims + COMMAND pio_decomp_extra_dims) + set_tests_properties(pio_decomp_extra_dims + PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) +else () + add_mpi_test(pio_decomp_extra_dims + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_extra_dims + NUMPROCS 4 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) +endif () + #===== pio_iosystems_test ===== add_executable (pio_iosystem_tests EXCLUDE_FROM_ALL pio_iosystem_tests.F90 diff --git a/tests/general/pio_decomp_extra_dims.F90.in b/tests/general/pio_decomp_extra_dims.F90.in new file mode 100644 index 0000000000..e36fc56374 --- /dev/null +++ b/tests/general/pio_decomp_extra_dims.F90.in @@ -0,0 +1,896 @@ +! Write a 1d non-record variable, use a 1d decomposition +! that has no extra outermost dimensions +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_no_extra_dims + implicit none + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: wbuf, rbuf + integer, dimension(1) :: dims + integer :: pio_dim + integer :: i, ierr + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + wbuf = pio_tf_world_rank_; + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + ! Write the variable out + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Cannot inq var " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + + PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) +PIO_TF_AUTO_TEST_SUB_END decomp_no_extra_dims + +! Write a 1d non-record variable, use a 2d decomposition +! that has one extra outermost dimension (length set to 1) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_1_extra_dim + implicit none + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: wbuf, rbuf + integer, dimension(2) :: dims + integer :: pio_dim + integer :: i, ierr + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + dims(2) = 1 + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + wbuf = pio_tf_world_rank_; + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + ! Write the variable out + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Cannot inq var " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + + PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) +PIO_TF_AUTO_TEST_SUB_END decomp_1_extra_dim + +! Write a 1d non-record variable, use a 3d decomposition +! that has two extra outermost dimensions (lengths set to 1) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_2_extra_dims + implicit none + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: wbuf, rbuf + integer, dimension(3) :: dims + integer :: pio_dim + integer :: i, ierr + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + dims(2) = 1 + dims(3) = 1 + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + wbuf = pio_tf_world_rank_; + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + ! Write the variable out + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Cannot inq var " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + + PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) +PIO_TF_AUTO_TEST_SUB_END decomp_2_extra_dims + +! Write multiple frames of a 2d record variable (with an +! unlimited time dimension), use a 1d decomposition that +! has no extra outermost dimensions +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_no_extra_dims_rec + implicit none + integer, parameter :: NFRAMES = 3 + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var' + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: wbuf, rbuf + integer, dimension(1) :: dims + integer, dimension(2) :: pio_dims + integer :: i, ierr, lsz + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements on each frame + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + + allocate(wbuf(VEC_LOCAL_SZ, NFRAMES)) + allocate(rbuf(VEC_LOCAL_SZ, NFRAMES)) + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + do f=1,NFRAMES + do i=1,VEC_LOCAL_SZ + wbuf(i,f) = compdof(i) + (f-1) * dims(1) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + end do + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Could not inq var : " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf(:,f), wbuf(:,f)), "Got wrong val, frame=", f) + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END decomp_no_extra_dims_rec + +! Write multiple frames of a 2d record variable (with an +! unlimited time dimension), use a 2d decomposition that +! has one extra outermost dimension (length set to 1) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_1_extra_dim_rec + implicit none + integer, parameter :: NFRAMES = 3 + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var' + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: wbuf, rbuf + integer, dimension(2) :: dims + integer, dimension(2) :: pio_dims + integer :: i, ierr, lsz + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements on each frame + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + dims(2) = 1 + + allocate(wbuf(VEC_LOCAL_SZ, NFRAMES)) + allocate(rbuf(VEC_LOCAL_SZ, NFRAMES)) + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + do f=1,NFRAMES + do i=1,VEC_LOCAL_SZ + wbuf(i,f) = compdof(i) + (f-1) * dims(1) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + end do + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Could not inq var : " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf(:,f), wbuf(:,f)), "Got wrong val, frame=", f) + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END decomp_1_extra_dim_rec + +! Write multiple frames of a 2d record variable (with an +! unlimited time dimension), use a 3d decomposition that +! has two extra outermost dimensions (lengths set to 1) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_2_extra_dims_rec + implicit none + integer, parameter :: NFRAMES = 3 + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var' + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: wbuf, rbuf + integer, dimension(3) :: dims + integer, dimension(2) :: pio_dims + integer :: i, ierr, lsz + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements on each frame + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + dims(2) = 1 + dims(3) = 1 + + allocate(wbuf(VEC_LOCAL_SZ, NFRAMES)) + allocate(rbuf(VEC_LOCAL_SZ, NFRAMES)) + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + do f=1,NFRAMES + do i=1,VEC_LOCAL_SZ + wbuf(i,f) = compdof(i) + (f-1) * dims(1) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + end do + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Could not inq var : " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf(:,f), wbuf(:,f)), "Got wrong val, frame=", f) + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END decomp_2_extra_dims_rec + +! Write multiple frames of a 2d quasi-record variable (with +! a limited time dimension), use a 1d decomposition that +! has no extra outermost dimensions +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_no_extra_dims_lim_rec + implicit none + integer, parameter :: NFRAMES = 3 + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var' + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: wbuf, rbuf + integer, dimension(1) :: dims + integer, dimension(2) :: pio_dims + integer :: i, ierr, lsz + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements on each frame + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + + allocate(wbuf(VEC_LOCAL_SZ, NFRAMES)) + allocate(rbuf(VEC_LOCAL_SZ, NFRAMES)) + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + do f=1,NFRAMES + do i=1,VEC_LOCAL_SZ + wbuf(i,f) = compdof(i) + (f-1) * dims(1) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', NFRAMES, pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + end do + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Could not inq var : " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf(:,f), wbuf(:,f)), "Got wrong val, frame=", f) + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END decomp_no_extra_dims_lim_rec + +! Write multiple frames of a 2d quasi-record variable (with +! a limited time dimension), use a 2d decomposition that +! has one extra outermost dimension (length set to 1) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_1_extra_dim_lim_rec + implicit none + integer, parameter :: NFRAMES = 3 + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var' + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: wbuf, rbuf + integer, dimension(2) :: dims + integer, dimension(2) :: pio_dims + integer :: i, ierr, lsz + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements on each frame + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + dims(2) = 1 + + allocate(wbuf(VEC_LOCAL_SZ, NFRAMES)) + allocate(rbuf(VEC_LOCAL_SZ, NFRAMES)) + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + do f=1,NFRAMES + do i=1,VEC_LOCAL_SZ + wbuf(i,f) = compdof(i) + (f-1) * dims(1) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', NFRAMES, pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + end do + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Could not inq var : " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf(:,f), wbuf(:,f)), "Got wrong val, frame=", f) + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END decomp_1_extra_dim_lim_rec + +! Write multiple frames of a 2d record quasi-variable (with +! a limited time dimension), use a 3d decomposition that +! has two extra outermost dimensions (lengths set to 1) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN decomp_2_extra_dims_lim_rec + implicit none + integer, parameter :: NFRAMES = 3 + integer, parameter :: VEC_LOCAL_SZ = 7 + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var' + type(io_desc_t) :: iodesc + integer, dimension(VEC_LOCAL_SZ) :: compdof + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: wbuf, rbuf + integer, dimension(3) :: dims + integer, dimension(2) :: pio_dims + integer :: i, ierr, lsz + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Data evenly distributed across all procs + ! Each proc has VEC_LOCAL_SZ elements on each frame + dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ + dims(2) = 1 + dims(3) = 1 + + allocate(wbuf(VEC_LOCAL_SZ, NFRAMES)) + allocate(rbuf(VEC_LOCAL_SZ, NFRAMES)) + + do i=1,VEC_LOCAL_SZ + compdof(i) = VEC_LOCAL_SZ * pio_tf_world_rank_ + i + end do + + do f=1,NFRAMES + do i=1,VEC_LOCAL_SZ + wbuf(i,f) = compdof(i) + (f-1) * dims(1) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', NFRAMES, pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var, iodesc, wbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + end do + +#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC + call PIO_closefile(pio_file) + + ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite) + PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename)) + + ierr = PIO_inq_varid(pio_file, 'PIO_TF_test_var', pio_var) + PIO_TF_CHECK_ERR(ierr, "Could not inq var : " // trim(filename)) +#else + call PIO_syncfile(pio_file) +#endif + + rbuf = 0 + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var, f) + call PIO_read_darray(pio_file, pio_var, iodesc, rbuf(:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf(:,f), wbuf(:,f)), "Got wrong val, frame=", f) + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END decomp_2_extra_dims_lim_rec From d0163d2fea76eeb7e30f57cbd60499a19f07f432 Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 13 Jun 2018 13:47:48 -0500 Subject: [PATCH 2/2] Handling extra dimensions in the decomposition Update existing PIO2 code to get matched start/count values from a decomposition that might have extra dimensions. For start/count arrays, correct some for loops on the condition part according to their lengths. Strictly speaking, record variables are ones that use unlimited time dimension (rec_var flag is set). However, some E3SM tests do handle "quasi-record" variables (use limited time dimension, thus rec_var flag is NOT set). If a variable has more than one dimension and its record number has been set, we assume that it is either a record variable or a "quasi-record" one (handled similarly). --- src/clib/pio_darray_int.c | 159 +++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 69 deletions(-) diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 9ef8b00da8..4c87a4b00a 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -75,13 +75,13 @@ int compute_buffer_init(iosystem_desc_t *ios) /** * Fill start/count arrays for write_darray_multi_par(). This is an - * internal funciton. + * internal function. * * @param ndims the number of dims in the decomposition. + * @param dimlen the lengths of dims in the decomposition. * @param fndims the number of dims in the file. * @param vdesc pointer to the var_desc_t info. * @param region pointer to a region. - * @param frame array of record values. * @param start an already-allocated array which gets the start * values. * @param count an already-allocated array which gets the count @@ -90,9 +90,8 @@ int compute_buffer_init(iosystem_desc_t *ios) * @ingroup PIO_write_darray * @author Ed Hartnett */ -int find_start_count(int ndims, int fndims, var_desc_t *vdesc, - io_region *region, const int *frame, size_t *start, - size_t *count) +int find_start_count(int ndims, const int *dimlen, int fndims, var_desc_t *vdesc, + io_region *region, size_t *start, size_t *count) { /* Init start/count arrays to zero. */ for (int i = 0; i < fndims; i++) @@ -103,42 +102,44 @@ int find_start_count(int ndims, int fndims, var_desc_t *vdesc, if (region) { - if (vdesc->record >= 0) + /* Allow extra outermost dimensions in the decomposition */ + int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (ndims - (fndims - 1)) : (ndims - fndims); + pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__); + if (num_extra_dims > 0) + { + for (int d = 0; d < num_extra_dims; d++) + pioassert(dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1", + __FILE__, __LINE__); + } + + if (vdesc->record >= 0 && fndims > 1) { /* This is a record based multidimensional * array. Figure out start/count for all but the * record dimension (dimid 0). */ - for (int i = fndims - ndims; i < fndims; i++) + for (int i = 1; i < fndims; i++) { - start[i] = region->start[i - (fndims - ndims)]; - count[i] = region->count[i - (fndims - ndims)]; + start[i] = region->start[num_extra_dims + (i - 1)]; + count[i] = region->count[num_extra_dims + (i - 1)]; } - /* Now figure out start/count for record dimension. */ - if (fndims > 1 && ndims < fndims && count[1] > 0) - { + /* Set count for record dimension (start cannot be determined so far). */ + if (count[1] > 0) count[0] = 1; - start[0] = frame[0]; - } - else if (fndims == ndims) - { - /* ??? */ - start[0] += vdesc->record; - } } else { /* This is a non record variable. */ - for (int i = 0; i < ndims; i++) + for (int i = 0; i < fndims; i++) { - start[i] = region->start[i]; - count[i] = region->count[i]; + start[i] = region->start[num_extra_dims + i]; + count[i] = region->count[num_extra_dims + i]; } } #if PIO_ENABLE_LOGGING /* Log arrays for debug purposes. */ - for (int i = 0; i < ndims; i++) + for (int i = 0; i < fndims; i++) LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i])); #endif /* PIO_ENABLE_LOGGING */ } @@ -214,7 +215,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * for (int regioncnt = 0; regioncnt < num_regions; regioncnt++) { /* Fill the start/count arrays. */ - if ((ierr = find_start_count(iodesc->ndims, fndims, vdesc, region, frame, start, count))) + if ((ierr = find_start_count(iodesc->ndims, iodesc->dimlen, fndims, vdesc, region, start, count))) return pio_err(ios, file, ierr, __FILE__, __LINE__); /* IO tasks will run the netCDF/pnetcdf functions to write the data. */ @@ -225,9 +226,8 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * /* For each variable to be written. */ for (int nv = 0; nv < nvars; nv++) { - /* Set the start of the record dimension. (Hasn't - * this already been set above ???) */ - if (vdesc->record >= 0 && ndims < fndims) + /* Set the start of the record dimension. */ + if (vdesc->record >= 0 && fndims > 1) start[0] = frame[nv]; /* If there is data for this region, get a pointer to it. */ @@ -329,9 +329,9 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * /* Get the var info. */ vdesc = file->varlist + varids[nv]; - /* If this is a record var, set the start for + /* If this is a record (or quasi-record) var, set the start for * the record dimension. */ - if (vdesc->record >= 0 && ndims < fndims) + if (vdesc->record >= 0 && fndims > 1) for (int rc = 0; rc < rrcnt; rc++) startlist[rc][0] = frame[nv]; @@ -402,6 +402,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * * @param maxregions the number of regions in the list. * @param fndims the number of dimensions in the file. * @param iodesc_ndims the number of dimensions in the decomposition. + * @param dimlen the lengths of dimensions in the decomposition. * @param vdesc pointer to an array of var_desc_t for the vars being * written. * @param tmp_start pointer to an already allocaed array of length @@ -415,8 +416,8 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * * @author Jim Edwards, Ed Hartnett **/ int find_all_start_count(io_region *region, int maxregions, int fndims, - int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start, - size_t *tmp_count) + int iodesc_ndims, const int *dimlen, var_desc_t *vdesc, + size_t *tmp_start, size_t *tmp_count) { /* Check inputs. */ pioassert(maxregions >= 0 && fndims > 0 && iodesc_ndims >= 0 && vdesc && @@ -434,15 +435,25 @@ int find_all_start_count(io_region *region, int maxregions, int fndims, if (region) { - if (vdesc->record >= 0) + /* Allow extra outermost dimensions in the decomposition */ + int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (iodesc_ndims - (fndims - 1)) : (iodesc_ndims - fndims); + pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__); + if (num_extra_dims > 0) + { + for (int d = 0; d < num_extra_dims; d++) + pioassert(dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1", + __FILE__, __LINE__); + } + + if (vdesc->record >= 0 && fndims > 1) { /* This is a record based multidimensional * array. Copy start/count for non-record * dimensions. */ - for (int i = fndims - iodesc_ndims; i < fndims; i++) + for (int i = 1; i < fndims; i++) { - tmp_start[i + r * fndims] = region->start[i - (fndims - iodesc_ndims)]; - tmp_count[i + r * fndims] = region->count[i - (fndims - iodesc_ndims)]; + tmp_start[i + r * fndims] = region->start[num_extra_dims + (i - 1)]; + tmp_count[i + r * fndims] = region->count[num_extra_dims + (i - 1)]; LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims, tmp_start[i + r * fndims], i + r * fndims, tmp_count[i + r * fndims])); @@ -451,10 +462,10 @@ int find_all_start_count(io_region *region, int maxregions, int fndims, else { /* This is not a record based multidimensional array. */ - for (int i = 0; i < iodesc_ndims; i++) + for (int i = 0; i < fndims; i++) { - tmp_start[i + r * fndims] = region->start[i]; - tmp_count[i + r * fndims] = region->count[i]; + tmp_start[i + r * fndims] = region->start[num_extra_dims + i]; + tmp_count[i + r * fndims] = region->count[num_extra_dims + i]; LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims, tmp_start[i + r * fndims], i + r * fndims, tmp_count[i + r * fndims])); @@ -650,25 +661,21 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame, for (int nv = 0; nv < nvars; nv++) { LOG((3, "writing buffer var %d", nv)); - vdesc = file->varlist + varids[0]; + vdesc = file->varlist + varids[nv]; /* Get a pointer to the correct part of the buffer. */ bufptr = (void *)((char *)iobuf + iodesc->mpitype_size * (nv * rlen + loffset)); - /* If this var has an unlimited dim, set + /* If this var has a record dim, set * the start on that dim to the frame * value for this variable. */ - if (vdesc->record >= 0) + if (vdesc->record >= 0 && fndims > 1) { - if (fndims > 1 && iodesc->ndims < fndims && count[1] > 0) + if (count[1] > 0) { count[0] = 1; start[0] = frame[nv]; } - else if (fndims == iodesc->ndims) - { - start[0] += vdesc->record; - } } /* Call the netCDF functions to write the data. */ @@ -804,7 +811,7 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in /* Fill the tmp_start and tmp_count arrays, which contain the * start and count arrays for all regions. */ - if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, vdesc, + if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, iodesc->dimlen, vdesc, tmp_start, tmp_count))) return pio_err(ios, file, ierr, __FILE__, __LINE__); @@ -879,10 +886,6 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid /* Get the number of dimensions in the decomposition. */ ndims = iodesc->ndims; - /* Is this a non-record var? */ - if (fndims == ndims) - vdesc->record = -1; - /* IO procs will actially read the data. */ if (ios->ioproc) { @@ -900,10 +903,10 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid the mpitype. */ region = iodesc->firstregion; - /* ??? */ + /* This is a record (or quasi-record) var. If the record + number has not been set yet, set it to 0 by default */ if (fndims > ndims) { - ndims++; if (vdesc->record < 0) vdesc->record = 0; } @@ -932,16 +935,26 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset)); + /* Allow extra outermost dimensions in the decomposition */ + int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (ndims - (fndims - 1)) : (ndims - fndims); + pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__); + if (num_extra_dims > 0) + { + for (int d = 0; d < num_extra_dims; d++) + pioassert(iodesc->dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1", + __FILE__, __LINE__); + } + /* Get the start/count arrays. */ if (vdesc->record >= 0 && fndims > 1) { - /* This is a record var. The unlimited dimension + /* This is a record (or quasi-record) var. The record dimension * (0) is handled specially. */ start[0] = vdesc->record; - for (int i = 1; i < ndims; i++) + for (int i = 1; i < fndims; i++) { - start[i] = region->start[i-1]; - count[i] = region->count[i-1]; + start[i] = region->start[num_extra_dims + (i - 1)]; + count[i] = region->count[num_extra_dims + (i - 1)]; } /* Read one record. */ @@ -951,10 +964,10 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid else { /* Non-time dependent array */ - for (int i = 0; i < ndims; i++) + for (int i = 0; i < fndims; i++) { - start[i] = region->start[i]; - count[i] = region->count[i]; + start[i] = region->start[num_extra_dims + i]; + count[i] = region->count[num_extra_dims + i]; } } } @@ -1114,10 +1127,6 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc, /* Get the number of dims in our decomposition. */ ndims = iodesc->ndims; - /* Is this a non-record var? */ - if (fndims == ndims) - vdesc->record = -1; - if (ios->ioproc) { io_region *region; @@ -1133,6 +1142,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc, the mpitype. */ region = iodesc->firstregion; + /* This is a record (or quasi-record) var. If the record + number has not been set yet, set it to 0 by default */ if (fndims > ndims) { if (vdesc->record < 0) @@ -1154,16 +1165,26 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc, } else { + /* Allow extra outermost dimensions in the decomposition */ + int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (ndims - (fndims - 1)) : (ndims - fndims); + pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__); + if (num_extra_dims > 0) + { + for (int d = 0; d < num_extra_dims; d++) + pioassert(iodesc->dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1", + __FILE__, __LINE__); + } + if (vdesc->record >= 0 && fndims > 1) { - /* This is a record var. Find start for record dims. */ + /* This is a record (or quasi-record) var. Find start for record dims. */ tmp_start[regioncnt * fndims] = vdesc->record; /* Find start/count for all non-record dims. */ for (int i = 1; i < fndims; i++) { - tmp_start[i + regioncnt * fndims] = region->start[i - 1]; - tmp_count[i + regioncnt * fndims] = region->count[i - 1]; + tmp_start[i + regioncnt * fndims] = region->start[num_extra_dims + (i - 1)]; + tmp_count[i + regioncnt * fndims] = region->count[num_extra_dims + (i - 1)]; } /* Set count for record dimension. */ @@ -1175,8 +1196,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc, /* Non-time dependent array */ for (int i = 0; i < fndims; i++) { - tmp_start[i + regioncnt * fndims] = region->start[i]; - tmp_count[i + regioncnt * fndims] = region->count[i]; + tmp_start[i + regioncnt * fndims] = region->start[num_extra_dims + i]; + tmp_count[i + regioncnt * fndims] = region->count[num_extra_dims + i]; } } }