Skip to content

Commit

Permalink
now can turn on deflate with netcdf-4 parallel
Browse files Browse the repository at this point in the history
  • Loading branch information
@edwardhartnett
edwardhartnett committed Jul 22, 2020
1 parent e7cdd55 commit 282e338
Show file tree
Hide file tree
Showing 6 changed files with 23 additions and 53 deletions.
7 changes: 2 additions & 5 deletions src/clib/pio_nc4.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,8 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
if (ios->ioproc)
{
#ifdef _NETCDF4
if (file->iotype == PIO_IOTYPE_NETCDF4P)
ierr = NC_EINVAL;
else
if (file->do_io)
ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
if (file->do_io)
ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
#endif
}

Expand Down
4 changes: 2 additions & 2 deletions tests/cunit/run_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ printf 'running PIO tests...\n'

PIO_TESTS='test_intercomm2 test_async_mpi test_spmd test_rearr test_async_simple '\
'test_async_3proc test_async_4proc test_iosystem2_simple test_iosystem2_simple2 '\
'test_iosystem2 test_iosystem3_simple test_iosystem3_simple2 test_iosystem3 test_pioc '\
'test_iosystem2 test_iosystem3_simple test_iosystem3_simple2 test_iosystem3 test_simple test_pioc '\
'test_pioc_unlim test_pioc_putget test_pioc_fill test_darray test_darray_multi '\
'test_darray_multivar test_darray_multivar2 test_darray_multivar3 test_darray_1d '\
'test_darray_3d test_decomp_uneven test_decomps test_darray_async_simple '\
'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '\
'test_darray_fill test_darray_vard test_async_1d test_simple'
'test_darray_fill test_darray_vard test_async_1d '

success1=true
success2=true
Expand Down
19 changes: 5 additions & 14 deletions tests/cunit/test_pioc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1570,18 +1570,9 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank)
if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)))
ERR(ret);

/* Setting deflate should not work with parallel iotype. */
ret = PIOc_def_var_deflate(ncid, 0, 0, 1, 1);
if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
{
if (ret == PIO_NOERR)
ERR(ERR_WRONG);
}
else
{
if (ret != PIO_NOERR)
ERR(ERR_WRONG);
}
/* Setting deflate works with parallel iotype starting with netcdf-c-4.7.4. */
if ((ret = PIOc_def_var_deflate(ncid, 0, 0, 1, 1)))
ERR(ret);

/* Check that the inq_varname function works. */
if ((ret = PIOc_inq_varname(ncid, 0, NULL)))
Expand Down Expand Up @@ -1611,9 +1602,9 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank)
if (shuffle || !deflate || deflate_level != 1)
ERR(ERR_AWFUL);

/* For parallel netCDF-4, no compression available. :-( */
/* For parallel netCDF-4, we turned on deflate above. */
if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
if (shuffle || deflate)
if (shuffle || !deflate || deflate_level != 1)
ERR(ERR_AWFUL);

/* Check setting the chunk cache for the variable. */
Expand Down
10 changes: 5 additions & 5 deletions tests/cunit/test_simple.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,11 @@ int main(int argc, char **argv)
if (ret != PIO_ENOTNC4)
ERR(ERR_WRONG);
}
else if (flavor[f] == PIO_IOTYPE_NETCDF4P)
{
if (ret != NC_EINVAL)
ERR(ERR_WRONG);
}
/* else if (flavor[f] == PIO_IOTYPE_NETCDF4P) */
/* { */
/* if (ret != NC_EINVAL) */
/* ERR(ERR_WRONG); */
/* } */
else
{
if (ret)
Expand Down
2 changes: 0 additions & 2 deletions tests/unit/basic_tests.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,15 +285,13 @@ Subroutine test_open(test_id, err_msg)
call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
end if

ret_val = PIO_set_log_level(3)
ret_val = PIO_inq_unlimdim(pio_file, unlimdimid)
if(unlimdimid /= -1) then
err_msg = "Error in inq_unlimdim"
call PIO_closefile(pio_file)
print *,__FILE__,__LINE__,iotype, trim(err_msg)
call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
end if
ret_val = PIO_set_log_level(0)

! Close file
call PIO_closefile(pio_file)
Expand Down
34 changes: 9 additions & 25 deletions tests/unit/ncdf_tests.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,12 +469,12 @@ Subroutine test_nc4(test_id, err_msg)
! deflate_level_2 = 4
deflate_level = 1
deflate_level_2 = 1
ret_val = PIO_set_log_level(3)
ret_val = PIO_def_var_deflate(pio_file, pio_var, shuffle, deflate, &
deflate_level)

! Should not have worked except for netCDF-4/HDF5 serial.
if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
! Should not have worked except for netCDF-4/HDF5.
if ((iotype .eq. PIO_iotype_netcdf4c .or. iotype .eq. PIO_iotype_netcdf4p) &
.and. ret_val .ne. PIO_NOERR) then
err_msg = "Could not turn on compression for variable foo2222"
call PIO_closefile(pio_file)
return
Expand All @@ -486,10 +486,6 @@ Subroutine test_nc4(test_id, err_msg)
err_msg = "Did not get expected error when trying to turn deflate on for netcdf classic file"
call PIO_closefile(pio_file)
return
else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .eq. PIO_NOERR) then
err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
call PIO_closefile(pio_file)
return
end if

print*, 'testing PIO_put_att'
Expand All @@ -514,8 +510,8 @@ Subroutine test_nc4(test_id, err_msg)
print*, 'testing PIO_inq_var_deflate'
ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, my_deflate_level)

! Should not have worked except for netCDF-4/HDF5 serial.
if (iotype .eq. PIO_iotype_netcdf4c) then
! Should not have worked except for netCDF-4/HDF5.
if (iotype .eq. PIO_iotype_netcdf4c .or. iotype .eq. PIO_iotype_netcdf4p) then
if (ret_val .ne. PIO_NOERR) then
err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
call PIO_closefile(pio_file)
Expand All @@ -532,26 +528,14 @@ Subroutine test_nc4(test_id, err_msg)
err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
call PIO_closefile(pio_file)
return
else if (iotype .eq. PIO_iotype_netcdf4p) then
if (ret_val .ne. PIO_NOERR) then
err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
call PIO_closefile(pio_file)
return
else
if (shuffle .ne. 0 .or. deflate .ne. 0) then
err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
call PIO_closefile(pio_file)
return
end if
end if
end if

! Try to turn on compression for this variable.
print*, 'testing PIO_def_var_deflate'
ret_val = PIO_def_var_deflate(pio_file, pio_var%varid, shuffle, deflate, &
deflate_level_2)

! Should not have worked except for netCDF-4/HDF5 serial.
! Should not have worked except for netCDF-4/HDF5.
if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
err_msg = "Could not turn on compression for variable foo2222 second time"
call PIO_closefile(pio_file)
Expand All @@ -564,8 +548,8 @@ Subroutine test_nc4(test_id, err_msg)
err_msg = "Did not get expected error when trying to turn deflate on for netcdf classic file"
call PIO_closefile(pio_file)
return
else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .eq. PIO_NOERR) then
err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
err_msg = "Could not turn on compression for variable foo2222 second time"
call PIO_closefile(pio_file)
return
end if
Expand Down Expand Up @@ -605,7 +589,7 @@ Subroutine test_nc4(test_id, err_msg)
call PIO_closefile(pio_file)
return
else
if (shuffle .ne. 0 .or. deflate .ne. 0) then
if (shuffle .ne. 0 .or. deflate .ne. 1 .or. my_deflate_level .ne. deflate_level_2) then
err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
call PIO_closefile(pio_file)
return
Expand Down

0 comments on commit 282e338

Please sign in to comment.