Skip to content

Commit

Permalink
CCPP tendencies bugfixes, global restart reproducibility, halo bounda…
Browse files Browse the repository at this point in the history 
…ry update in dycore (NCAR#178)

* contributions from @SMoorthi-emc to fix the global restart reproducibility and to keep compiling without CCPP
* updates the submodule pointers for GFDL_atmos_cubed_sphere and ccpp-physics
* bugfix in ccpp/CMakeLists.txt to correctly set AVX2 flags or not (discovered by Yunheng)
* changes mod_name of non-phys tendencies in GFS_diagnostics.F90 to gfs_dyn from gfs_phys (from @grantfirl)
  • Loading branch information
@climbfuji
climbfuji authored Sep 30, 2020
1 parent 9e1ba7c commit ab0e5ae
Show file tree
Hide file tree
Showing 10 changed files with 169 additions and 58 deletions.
2 changes: 1 addition & 1 deletion atmos_cubed_sphere
Submodule atmos_cubed_sphere updated 15 files
+19 −0 docs/AdvectionOperators.md
+35 −0 docs/Chapter1.md
+110 −0 docs/Chapter2.md
+63 −0 docs/Chapter3.md
+65 −0 docs/Chapter4.md
+300 −0 docs/Doxyfile
+194 −0 docs/DoxygenLayout.xml
+25 −0 docs/Links.md
+26 −0 docs/Preface.md
+73 −0 docs/appendixB.md
+34 −0 docs/appendixB1.md
+22 −0 docs/appendixB2.md
+12 −0 docs/documentation.dox
+395 −0 model/fv_control.F90
+13 −1 model/fv_regional_bc.F90
21 changes: 11 additions & 10 deletions atmos_model.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,13 @@ module atmos_model_mod
IPD_interstitial => GFS_interstitial
use IPD_driver, only: IPD_initialize, IPD_initialize_rst
use CCPP_driver, only: CCPP_step, non_uniform_blocks

use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper
#else
use IPD_driver, only: IPD_initialize, IPD_initialize_rst, IPD_step
use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2
#endif

use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper

use FV3GFS_io_mod, only: FV3GFS_restart_read, FV3GFS_restart_write, &
FV3GFS_IPD_checksum, &
FV3GFS_diag_register, FV3GFS_diag_output, &
Expand Down Expand Up @@ -291,16 +291,16 @@ subroutine update_atmos_radiation_physics (Atmos)
#ifdef CCPP
call CCPP_step (step="time_vary", nblks=Atm_block%nblks, ierr=ierr)
if (ierr/=0) call mpp_error(FATAL, 'Call to CCPP time_vary step failed')

!--- call stochastic physics pattern generation / cellular automata
call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
if (ierr/=0) call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')

#else
Func1d => time_vary_step
call IPD_step (IPD_Control, IPD_Data(:), IPD_Diag, IPD_Restart, IPD_func1d=Func1d)
#endif

!--- call stochastic physics pattern generation / cellular automata
call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
if (ierr/=0) call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')


!--- if coupled, assign coupled fields

if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then
Expand Down Expand Up @@ -625,14 +625,15 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
#ifdef CCPP
call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, &
IPD_Interstitial, commglobal, mpp_npes(), Init_parm)
#else
call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
#endif

!--- Initialize stochastic physics pattern generation / cellular automata for first time step
call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
if (ierr/=0) call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')

#else
call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
#endif

Atmos%Diag => IPD_Diag

Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb
Expand Down
12 changes: 7 additions & 5 deletions ccpp/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,13 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel")
endif (LEGACY_INTEL)
elseif (${CMAKE_BUILD_TYPE} MATCHES "Release")
# Specify aggressive optimization flags (to be overwritten for individual files in ccpp-physics' CMakeLists.txt)
if (SIMDMULTIARCH)
set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512")
else (SIMDMULTIARCH)
set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
endif (SIMDMULTIARCH)
if (AVX2)
if (SIMDMULTIARCH)
set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512")
else (SIMDMULTIARCH)
set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
endif (SIMDMULTIARCH)
endif (AVX2)
set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3")
endif (${CMAKE_BUILD_TYPE} MATCHES "Debug")
set (CMAKE_Fortran_FLAGS_DEFAULT_PREC "-i4 -real-size 64")
Expand Down
12 changes: 6 additions & 6 deletions ccpp/driver/CCPP_driver.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ subroutine CCPP_step (step, nblks, ierr)
end do
end do

else if (trim(step)=="physics_init") then
else if (trim(step)=="physics_init") then

! Since the physics init steps are independent of the blocking structure,
! we can use cdata_domain here. Since we don't use threading on the outside,
Expand All @@ -107,7 +107,7 @@ subroutine CCPP_step (step, nblks, ierr)
return
end if

else if (trim(step)=="time_vary") then
else if (trim(step)=="time_vary") then

! Since the time_vary steps only use data structures for all blocks (except the
! CCPP-internal variables ccpp_error_flag and ccpp_error_message, which are defined
Expand All @@ -123,8 +123,8 @@ subroutine CCPP_step (step, nblks, ierr)
return
end if

! Radiation and stochastic physics
else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then
! Radiation and stochastic physics
else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then

! Set number of threads available to physics schemes to one,
! because threads are used on the outside for blocking
Expand Down Expand Up @@ -162,8 +162,8 @@ subroutine CCPP_step (step, nblks, ierr)
!$OMP end parallel
if (ierr/=0) return

! Finalize
else if (trim(step)=="finalize") then
! Finalize
else if (trim(step)=="finalize") then

! Loop over blocks, don't use threading on the outside but allowing threading
! inside the finalization, similar to what is done for the initialization
Expand Down
2 changes: 1 addition & 1 deletion ccpp/physics
Submodule physics updated 19 files
+3 −1 CMakeLists.txt
+12 −8 physics/cu_gf_driver.F90
+24 −2 physics/module_MYJPBL_wrapper.F90
+53 −0 physics/module_MYJPBL_wrapper.meta
+20 −48 physics/module_MYNNPBL_wrapper.F90
+13 −5 physics/module_MYNNPBL_wrapper.meta
+1 −1 physics/module_bl_mynn.F90
+5 −10 physics/moninedmf.f
+5 −13 physics/moninedmf.meta
+45 −2 physics/moninshoc.f
+70 −0 physics/moninshoc.meta
+28 −12 physics/satmedmfvdif.F
+13 −5 physics/satmedmfvdif.meta
+8 −8 physics/satmedmfvdifq.F
+13 −5 physics/satmedmfvdifq.meta
+44 −2 physics/shinhongvdif.F90
+76 −0 physics/shinhongvdif.meta
+44 −2 physics/ysuvdif.F90
+76 −0 physics/ysuvdif.meta
89 changes: 89 additions & 0 deletions ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
<?xml version="1.0" encoding="UTF-8"?>

<suite name="FV3_GFS_2017_couplednsst" lib="ccppphys" ver="4">
<!-- <init></init> -->
<group name="time_vary">
<subcycle loop="1">
<scheme>GFS_time_vary_pre</scheme>
<scheme>GFS_rrtmg_setup</scheme>
<scheme>GFS_rad_time_vary</scheme>
<scheme>GFS_phys_time_vary</scheme>
</subcycle>
</group>
<group name="radiation">
<subcycle loop="1">
<scheme>GFS_suite_interstitial_rad_reset</scheme>
<scheme>GFS_rrtmg_pre</scheme>
<scheme>rrtmg_sw_pre</scheme>
<scheme>rrtmg_sw</scheme>
<scheme>rrtmg_sw_post</scheme>
<scheme>rrtmg_lw_pre</scheme>
<scheme>rrtmg_lw</scheme>
<scheme>rrtmg_lw_post</scheme>
<scheme>GFS_rrtmg_post</scheme>
</subcycle>
</group>
<group name="physics">
<subcycle loop="1">
<scheme>GFS_suite_interstitial_phys_reset</scheme>
<scheme>GFS_suite_stateout_reset</scheme>
<scheme>get_prs_fv3</scheme>
<scheme>GFS_suite_interstitial_1</scheme>
<scheme>GFS_surface_generic_pre</scheme>
<scheme>GFS_surface_composites_pre</scheme>
<scheme>dcyc2t3</scheme>
<scheme>GFS_surface_composites_inter</scheme>
<scheme>GFS_suite_interstitial_2</scheme>
</subcycle>
<!-- Surface iteration loop -->
<subcycle loop="2">
<scheme>sfc_diff</scheme>
<scheme>GFS_surface_loop_control_part1</scheme>
<scheme>lsm_noah</scheme>
<scheme>sfc_nst_pre</scheme>
<scheme>sfc_nst</scheme>
<scheme>sfc_nst_post</scheme>
<scheme>sfc_cice</scheme>
<scheme>sfc_sice</scheme>
<scheme>GFS_surface_loop_control_part2</scheme>
</subcycle>
<!-- End of surface iteration loop -->
<subcycle loop="1">
<scheme>GFS_surface_composites_post</scheme>
<scheme>sfc_diag</scheme>
<scheme>sfc_diag_post</scheme>
<scheme>GFS_surface_generic_post</scheme>
<scheme>GFS_PBL_generic_pre</scheme>
<scheme>hedmf</scheme>
<scheme>GFS_PBL_generic_post</scheme>
<scheme>GFS_GWD_generic_pre</scheme>
<scheme>cires_ugwp</scheme>
<scheme>cires_ugwp_post</scheme>
<scheme>GFS_GWD_generic_post</scheme>
<scheme>rayleigh_damp</scheme>
<scheme>GFS_suite_stateout_update</scheme>
<scheme>ozphys</scheme>
<scheme>get_phi_fv3</scheme>
<scheme>GFS_suite_interstitial_3</scheme>
<scheme>GFS_DCNV_generic_pre</scheme>
<scheme>samfdeepcnv</scheme>
<scheme>GFS_DCNV_generic_post</scheme>
<scheme>GFS_SCNV_generic_pre</scheme>
<scheme>samfshalcnv</scheme>
<scheme>GFS_SCNV_generic_post</scheme>
<scheme>GFS_suite_interstitial_4</scheme>
<scheme>cnvc90</scheme>
<scheme>GFS_MP_generic_pre</scheme>
<scheme>zhaocarr_gscond</scheme>
<scheme>zhaocarr_precpd</scheme>
<scheme>GFS_MP_generic_post</scheme>
<scheme>maximum_hourly_diagnostics</scheme>
</subcycle>
</group>
<group name="stochastics">
<subcycle loop="1">
<scheme>GFS_stochastics</scheme>
</subcycle>
</group>
<!-- <finalize></finalize> -->
</suite>
2 changes: 1 addition & 1 deletion ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>

<suite name="FV3_GFS_cpld_rasmgshocnsst" lib="ccppphys" ver="3">
<suite name="FV3_GFS_cpld_rasmgshocnsst" lib="ccppphys" ver="4">
<!-- <init></init> -->
<group name="time_vary">
<subcycle loop="1">
Expand Down
10 changes: 5 additions & 5 deletions gfsphysics/GFS_layer/GFS_diagnostics.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2445,7 +2445,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
ExtDiag(idx)%name = 'dt3dt_nophys'
ExtDiag(idx)%desc = 'temperature tendency due to non-physics processes'
ExtDiag(idx)%unit = 'K s-1'
ExtDiag(idx)%mod_name = 'gfs_phys'
ExtDiag(idx)%mod_name = 'gfs_dyn'
ExtDiag(idx)%time_avg = .TRUE.
allocate (ExtDiag(idx)%data(nblks))
do nb = 1,nblks
Expand Down Expand Up @@ -2626,7 +2626,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
ExtDiag(idx)%name = 'du3dt_nophys'
ExtDiag(idx)%desc = 'u momentum tendency due to non-physics processes'
ExtDiag(idx)%unit = 'm s-2'
ExtDiag(idx)%mod_name = 'gfs_phys'
ExtDiag(idx)%mod_name = 'gfs_dyn'
ExtDiag(idx)%time_avg = .TRUE.
allocate (ExtDiag(idx)%data(nblks))
do nb = 1,nblks
Expand All @@ -2638,7 +2638,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
ExtDiag(idx)%name = 'dv3dt_nophys'
ExtDiag(idx)%desc = 'v momentum tendency due to non-physics processes'
ExtDiag(idx)%unit = 'm s-2'
ExtDiag(idx)%mod_name = 'gfs_phys'
ExtDiag(idx)%mod_name = 'gfs_dyn'
ExtDiag(idx)%time_avg = .TRUE.
allocate (ExtDiag(idx)%data(nblks))
do nb = 1,nblks
Expand Down Expand Up @@ -2785,7 +2785,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
ExtDiag(idx)%name = 'dq3dt_nophys'
ExtDiag(idx)%desc = 'water vapor specific humidity tendency due to non-physics processes'
ExtDiag(idx)%unit = 'kg kg-1 s-1'
ExtDiag(idx)%mod_name = 'gfs_phys'
ExtDiag(idx)%mod_name = 'gfs_dyn'
ExtDiag(idx)%time_avg = .TRUE.
allocate (ExtDiag(idx)%data(nblks))
do nb = 1,nblks
Expand All @@ -2797,7 +2797,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
ExtDiag(idx)%name = 'dq3dt_o3nophys'
ExtDiag(idx)%desc = 'ozone concentration tendency due to non-physics processes'
ExtDiag(idx)%unit = 'kg kg-1 s-1'
ExtDiag(idx)%mod_name = 'gfs_phys'
ExtDiag(idx)%mod_name = 'gfs_dyn'
ExtDiag(idx)%time_avg = .TRUE.
allocate (ExtDiag(idx)%data(nblks))
do nb = 1,nblks
Expand Down
4 changes: 2 additions & 2 deletions gfsphysics/GFS_layer/GFS_typedefs.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1551,9 +1551,9 @@ module GFS_typedefs
#ifdef CCPP
real (kind=kind_phys), pointer :: TRAIN (:,:) => null() !< accumulated stratiform T tendency (K s-1)
#endif
#ifdef CCPP
!#ifdef CCPP
real (kind=kind_phys), pointer :: cldfra (:,:) => null() !< instantaneous 3D cloud fraction
#endif
!#endif
!--- MP quantities for 3D diagnositics
real (kind=kind_phys), pointer :: refl_10cm(:,:) => null() !< instantaneous refl_10cm
!
Expand Down
73 changes: 46 additions & 27 deletions io/FV3GFS_io.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1077,16 +1077,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain)

if (Sfcprop(nb)%lakefrac(ix) > zero) then
Sfcprop(nb)%oceanfrac(ix) = zero ! lake & ocean don't coexist in a cell
if (Sfcprop(nb)%fice(ix) < Model%min_lakeice) then
Sfcprop(nb)%fice(ix) = zero
if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
endif
! if (Sfcprop(nb)%fice(ix) < Model%min_lakeice) then
! Sfcprop(nb)%fice(ix) = zero
! if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
! endif
else
Sfcprop(nb)%oceanfrac(ix) = one - Sfcprop(nb)%landfrac(ix)
if (Sfcprop(nb)%fice(ix) < Model%min_seaice) then
Sfcprop(nb)%fice(ix) = zero
if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
endif
! if (Sfcprop(nb)%fice(ix) < Model%min_seaice) then
! Sfcprop(nb)%fice(ix) = zero
! if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
! endif
endif
!
!--- NSSTM variables
Expand Down Expand Up @@ -1357,28 +1357,47 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain)
enddo
enddo
else
if( Model%phour < 1.e-7) then
!$omp parallel do default(shared) private(nb, ix, tem)
do nb = 1, Atm_block%nblks
do ix = 1, Atm_block%blksz(nb)
do nb = 1, Atm_block%nblks
do ix = 1, Atm_block%blksz(nb)
!--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlo
! Sfcprop(nb)%tsfcl(ix) = Sfcprop(nb)%tsfco(ix)
! Sfcprop(nb)%zorll(ix) = Sfcprop(nb)%zorlo(ix)
! Sfcprop(nb)%zorli(ix) = Sfcprop(nb)%zorlo(ix)
! Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorlo(ix)
! Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfco(ix)
if (Sfcprop(nb)%slmsk(ix) == 1) then
Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix)
Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
else
tem = one - Sfcprop(nb)%fice(ix)
Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) &
+ Sfcprop(nb)%zorlo(ix) * tem

Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tisfc(ix) * Sfcprop(nb)%fice(ix) &
+ Sfcprop(nb)%tsfco(ix) * tem
endif
! Sfcprop(nb)%tsfcl(ix) = Sfcprop(nb)%tsfco(ix)
! Sfcprop(nb)%zorll(ix) = Sfcprop(nb)%zorlo(ix)
! Sfcprop(nb)%zorli(ix) = Sfcprop(nb)%zorlo(ix)
! Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorlo(ix)
if (Sfcprop(nb)%slmsk(ix) == 1) then
Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix)
Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
else
tem = one - Sfcprop(nb)%fice(ix)
Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) &
+ Sfcprop(nb)%zorlo(ix) * tem
Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tisfc(ix) * Sfcprop(nb)%fice(ix) &
+ Sfcprop(nb)%tsfco(ix) * tem
endif
enddo
enddo
enddo
else
!$omp parallel do default(shared) private(nb, ix, tem)
do nb = 1, Atm_block%nblks
do ix = 1, Atm_block%blksz(nb)
!--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlo
Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfco(ix)
if (Sfcprop(nb)%slmsk(ix) == 1) then
Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix)
Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
else
tem = one - Sfcprop(nb)%fice(ix)
Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) &
+ Sfcprop(nb)%zorlo(ix) * tem
if (Sfcprop(nb)%fice(ix) > min(Model%min_seaice,Model%min_lakeice)) then
Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
endif
endif
enddo
enddo
endif
endif ! if (Model%frac_grid)

!#ifdef CCPP
Expand Down

0 comments on commit ab0e5ae

Please sign in to comment.