From ab0e5ae33b24c191f201edb80361de799816a2d8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 30 Sep 2020 13:29:46 -0600 Subject: [PATCH] CCPP tendencies bugfixes, global restart reproducibility, halo boundary update in dycore (#178) * contributions from @SMoorthi-emc to fix the global restart reproducibility and to keep compiling without CCPP * updates the submodule pointers for GFDL_atmos_cubed_sphere and ccpp-physics * bugfix in ccpp/CMakeLists.txt to correctly set AVX2 flags or not (discovered by Yunheng) * changes mod_name of non-phys tendencies in GFS_diagnostics.F90 to gfs_dyn from gfs_phys (from @grantfirl) --- atmos_cubed_sphere | 2 +- atmos_model.F90 | 21 ++--- ccpp/CMakeLists.txt | 12 +-- ccpp/driver/CCPP_driver.F90 | 12 +-- ccpp/physics | 2 +- .../suites/suite_FV3_GFS_2017_couplednsst.xml | 89 +++++++++++++++++++ .../suite_FV3_GFS_cpld_rasmgshocnsst.xml | 2 +- gfsphysics/GFS_layer/GFS_diagnostics.F90 | 10 +-- gfsphysics/GFS_layer/GFS_typedefs.F90 | 4 +- io/FV3GFS_io.F90 | 73 +++++++++------ 10 files changed, 169 insertions(+), 58 deletions(-) create mode 100644 ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 8b59ebc03..2ec76f886 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 8b59ebc039dafe1c20ed6dd21cb38ca564852b98 +Subproject commit 2ec76f886450b1c58d2f7eb18f0553a1e77fb831 diff --git a/atmos_model.F90 b/atmos_model.F90 index 0499c3c56..81589c386 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -99,13 +99,13 @@ module atmos_model_mod IPD_interstitial => GFS_interstitial use IPD_driver, only: IPD_initialize, IPD_initialize_rst use CCPP_driver, only: CCPP_step, non_uniform_blocks + +use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper #else use IPD_driver, only: IPD_initialize, IPD_initialize_rst, IPD_step use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2 #endif -use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper - use FV3GFS_io_mod, only: FV3GFS_restart_read, FV3GFS_restart_write, & FV3GFS_IPD_checksum, & FV3GFS_diag_register, FV3GFS_diag_output, & @@ -291,16 +291,16 @@ subroutine update_atmos_radiation_physics (Atmos) #ifdef CCPP call CCPP_step (step="time_vary", nblks=Atm_block%nblks, ierr=ierr) if (ierr/=0) call mpp_error(FATAL, 'Call to CCPP time_vary step failed') + +!--- call stochastic physics pattern generation / cellular automata + call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr) + if (ierr/=0) call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed') + #else Func1d => time_vary_step call IPD_step (IPD_Control, IPD_Data(:), IPD_Diag, IPD_Restart, IPD_func1d=Func1d) #endif -!--- call stochastic physics pattern generation / cellular automata - call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr) - if (ierr/=0) call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed') - - !--- if coupled, assign coupled fields if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then @@ -625,14 +625,15 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) #ifdef CCPP call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, & IPD_Interstitial, commglobal, mpp_npes(), Init_parm) -#else - call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm) -#endif !--- Initialize stochastic physics pattern generation / cellular automata for first time step call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr) if (ierr/=0) call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed') +#else + call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm) +#endif + Atmos%Diag => IPD_Diag Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb diff --git a/ccpp/CMakeLists.txt b/ccpp/CMakeLists.txt index b1395b23b..750ae5c14 100644 --- a/ccpp/CMakeLists.txt +++ b/ccpp/CMakeLists.txt @@ -120,11 +120,13 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") endif (LEGACY_INTEL) elseif (${CMAKE_BUILD_TYPE} MATCHES "Release") # Specify aggressive optimization flags (to be overwritten for individual files in ccpp-physics' CMakeLists.txt) - if (SIMDMULTIARCH) - set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512") - else (SIMDMULTIARCH) - set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2") - endif (SIMDMULTIARCH) + if (AVX2) + if (SIMDMULTIARCH) + set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512") + else (SIMDMULTIARCH) + set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2") + endif (SIMDMULTIARCH) + endif (AVX2) set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") endif (${CMAKE_BUILD_TYPE} MATCHES "Debug") set (CMAKE_Fortran_FLAGS_DEFAULT_PREC "-i4 -real-size 64") diff --git a/ccpp/driver/CCPP_driver.F90 b/ccpp/driver/CCPP_driver.F90 index 8e45d9382..89c41672f 100644 --- a/ccpp/driver/CCPP_driver.F90 +++ b/ccpp/driver/CCPP_driver.F90 @@ -93,7 +93,7 @@ subroutine CCPP_step (step, nblks, ierr) end do end do - else if (trim(step)=="physics_init") then + else if (trim(step)=="physics_init") then ! Since the physics init steps are independent of the blocking structure, ! we can use cdata_domain here. Since we don't use threading on the outside, @@ -107,7 +107,7 @@ subroutine CCPP_step (step, nblks, ierr) return end if - else if (trim(step)=="time_vary") then + else if (trim(step)=="time_vary") then ! Since the time_vary steps only use data structures for all blocks (except the ! CCPP-internal variables ccpp_error_flag and ccpp_error_message, which are defined @@ -123,8 +123,8 @@ subroutine CCPP_step (step, nblks, ierr) return end if - ! Radiation and stochastic physics - else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then + ! Radiation and stochastic physics + else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then ! Set number of threads available to physics schemes to one, ! because threads are used on the outside for blocking @@ -162,8 +162,8 @@ subroutine CCPP_step (step, nblks, ierr) !$OMP end parallel if (ierr/=0) return - ! Finalize - else if (trim(step)=="finalize") then + ! Finalize + else if (trim(step)=="finalize") then ! Loop over blocks, don't use threading on the outside but allowing threading ! inside the finalization, similar to what is done for the initialization diff --git a/ccpp/physics b/ccpp/physics index f91d1bfde..6f12e1482 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit f91d1bfde0bcdf69a8efe0917f49a4713e590bef +Subproject commit 6f12e1482f7f1b2f99d9b6019d94486840c923c8 diff --git a/ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml b/ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml new file mode 100644 index 000000000..1aa7ca484 --- /dev/null +++ b/ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml @@ -0,0 +1,89 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + rrtmg_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + lsm_noah + sfc_nst_pre + sfc_nst + sfc_nst_post + sfc_cice + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + hedmf + GFS_PBL_generic_post + GFS_GWD_generic_pre + cires_ugwp + cires_ugwp_post + GFS_GWD_generic_post + rayleigh_damp + GFS_suite_stateout_update + ozphys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + zhaocarr_gscond + zhaocarr_precpd + GFS_MP_generic_post + maximum_hourly_diagnostics + + + + + GFS_stochastics + + + + diff --git a/ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml b/ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml index 5b3b63528..a08956dfa 100644 --- a/ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml +++ b/ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml @@ -1,6 +1,6 @@ - + diff --git a/gfsphysics/GFS_layer/GFS_diagnostics.F90 b/gfsphysics/GFS_layer/GFS_diagnostics.F90 index 1b6fabe96..adb624cca 100644 --- a/gfsphysics/GFS_layer/GFS_diagnostics.F90 +++ b/gfsphysics/GFS_layer/GFS_diagnostics.F90 @@ -2445,7 +2445,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%name = 'dt3dt_nophys' ExtDiag(idx)%desc = 'temperature tendency due to non-physics processes' ExtDiag(idx)%unit = 'K s-1' - ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%mod_name = 'gfs_dyn' ExtDiag(idx)%time_avg = .TRUE. allocate (ExtDiag(idx)%data(nblks)) do nb = 1,nblks @@ -2626,7 +2626,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%name = 'du3dt_nophys' ExtDiag(idx)%desc = 'u momentum tendency due to non-physics processes' ExtDiag(idx)%unit = 'm s-2' - ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%mod_name = 'gfs_dyn' ExtDiag(idx)%time_avg = .TRUE. allocate (ExtDiag(idx)%data(nblks)) do nb = 1,nblks @@ -2638,7 +2638,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%name = 'dv3dt_nophys' ExtDiag(idx)%desc = 'v momentum tendency due to non-physics processes' ExtDiag(idx)%unit = 'm s-2' - ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%mod_name = 'gfs_dyn' ExtDiag(idx)%time_avg = .TRUE. allocate (ExtDiag(idx)%data(nblks)) do nb = 1,nblks @@ -2785,7 +2785,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%name = 'dq3dt_nophys' ExtDiag(idx)%desc = 'water vapor specific humidity tendency due to non-physics processes' ExtDiag(idx)%unit = 'kg kg-1 s-1' - ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%mod_name = 'gfs_dyn' ExtDiag(idx)%time_avg = .TRUE. allocate (ExtDiag(idx)%data(nblks)) do nb = 1,nblks @@ -2797,7 +2797,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%name = 'dq3dt_o3nophys' ExtDiag(idx)%desc = 'ozone concentration tendency due to non-physics processes' ExtDiag(idx)%unit = 'kg kg-1 s-1' - ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%mod_name = 'gfs_dyn' ExtDiag(idx)%time_avg = .TRUE. allocate (ExtDiag(idx)%data(nblks)) do nb = 1,nblks diff --git a/gfsphysics/GFS_layer/GFS_typedefs.F90 b/gfsphysics/GFS_layer/GFS_typedefs.F90 index 67d8a7668..5fecfbfff 100644 --- a/gfsphysics/GFS_layer/GFS_typedefs.F90 +++ b/gfsphysics/GFS_layer/GFS_typedefs.F90 @@ -1551,9 +1551,9 @@ module GFS_typedefs #ifdef CCPP real (kind=kind_phys), pointer :: TRAIN (:,:) => null() !< accumulated stratiform T tendency (K s-1) #endif -#ifdef CCPP +!#ifdef CCPP real (kind=kind_phys), pointer :: cldfra (:,:) => null() !< instantaneous 3D cloud fraction -#endif +!#endif !--- MP quantities for 3D diagnositics real (kind=kind_phys), pointer :: refl_10cm(:,:) => null() !< instantaneous refl_10cm ! diff --git a/io/FV3GFS_io.F90 b/io/FV3GFS_io.F90 index e0898c3f6..458605c96 100644 --- a/io/FV3GFS_io.F90 +++ b/io/FV3GFS_io.F90 @@ -1077,16 +1077,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain) if (Sfcprop(nb)%lakefrac(ix) > zero) then Sfcprop(nb)%oceanfrac(ix) = zero ! lake & ocean don't coexist in a cell - if (Sfcprop(nb)%fice(ix) < Model%min_lakeice) then - Sfcprop(nb)%fice(ix) = zero - if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0 - endif +! if (Sfcprop(nb)%fice(ix) < Model%min_lakeice) then +! Sfcprop(nb)%fice(ix) = zero +! if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0 +! endif else Sfcprop(nb)%oceanfrac(ix) = one - Sfcprop(nb)%landfrac(ix) - if (Sfcprop(nb)%fice(ix) < Model%min_seaice) then - Sfcprop(nb)%fice(ix) = zero - if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0 - endif +! if (Sfcprop(nb)%fice(ix) < Model%min_seaice) then +! Sfcprop(nb)%fice(ix) = zero +! if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0 +! endif endif ! !--- NSSTM variables @@ -1357,28 +1357,47 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain) enddo enddo else + if( Model%phour < 1.e-7) then !$omp parallel do default(shared) private(nb, ix, tem) - do nb = 1, Atm_block%nblks - do ix = 1, Atm_block%blksz(nb) + do nb = 1, Atm_block%nblks + do ix = 1, Atm_block%blksz(nb) !--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlo -! Sfcprop(nb)%tsfcl(ix) = Sfcprop(nb)%tsfco(ix) -! Sfcprop(nb)%zorll(ix) = Sfcprop(nb)%zorlo(ix) -! Sfcprop(nb)%zorli(ix) = Sfcprop(nb)%zorlo(ix) -! Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorlo(ix) -! Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfco(ix) - if (Sfcprop(nb)%slmsk(ix) == 1) then - Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix) - Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix) - else - tem = one - Sfcprop(nb)%fice(ix) - Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) & - + Sfcprop(nb)%zorlo(ix) * tem - - Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tisfc(ix) * Sfcprop(nb)%fice(ix) & - + Sfcprop(nb)%tsfco(ix) * tem - endif +! Sfcprop(nb)%tsfcl(ix) = Sfcprop(nb)%tsfco(ix) +! Sfcprop(nb)%zorll(ix) = Sfcprop(nb)%zorlo(ix) +! Sfcprop(nb)%zorli(ix) = Sfcprop(nb)%zorlo(ix) +! Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorlo(ix) + if (Sfcprop(nb)%slmsk(ix) == 1) then + Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix) + Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix) + else + tem = one - Sfcprop(nb)%fice(ix) + Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) & + + Sfcprop(nb)%zorlo(ix) * tem + Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tisfc(ix) * Sfcprop(nb)%fice(ix) & + + Sfcprop(nb)%tsfco(ix) * tem + endif + enddo enddo - enddo + else +!$omp parallel do default(shared) private(nb, ix, tem) + do nb = 1, Atm_block%nblks + do ix = 1, Atm_block%blksz(nb) + !--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlo + Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfco(ix) + if (Sfcprop(nb)%slmsk(ix) == 1) then + Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix) + Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix) + else + tem = one - Sfcprop(nb)%fice(ix) + Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) & + + Sfcprop(nb)%zorlo(ix) * tem + if (Sfcprop(nb)%fice(ix) > min(Model%min_seaice,Model%min_lakeice)) then + Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix) + endif + endif + enddo + enddo + endif endif ! if (Model%frac_grid) !#ifdef CCPP