Merge pull request #94 from grantfirl/ufs-dev-PR49

UFS-dev PR#49
NCAR · Apr 18, 2023 · 2b61d7f · 2b61d7f
2 parents c5242e0 + 2af1836
commit 2b61d7f
Show file tree

Hide file tree

Showing 29 changed files with 3,883 additions and 2,551 deletions.
diff --git a/CICE-interface/CICE b/CICE-interface/CICE
diff --git a/CMEPS-interface/CMEPS b/CMEPS-interface/CMEPS
diff --git a/FV3 b/FV3
diff --git a/WW3 b/WW3
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -300,6 +300,7 @@ export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export RESTART_INTERVAL=0
 export QUILTING=.true.
+export QUILTING_RESTART=.false.
 export WRITE_GROUP=1
 export WRTTASK_PER_GROUP=6
 export ITASKS=1

diff --git a/tests/fv3_conf/control_run.IN b/tests/fv3_conf/control_run.IN
@@ -136,10 +136,10 @@ fi
 
 #Thompson
 if [ $IMP_PHYSICS = 8 ]; then
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qgV2.dat .
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN .
 fi
 
 #prognostic aerosols

diff --git a/tests/fv3_conf/cpld_control_run.IN b/tests/fv3_conf/cpld_control_run.IN
@@ -167,10 +167,10 @@ cp    @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127/oro_data_ls* ./INPUT
 cp    @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127/oro_data_ss* ./INPUT
 
 if [ $IMP_PHYSICS = 8 ]; then
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN CCN_ACTIVATE.BIN
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat freezeH2O.dat
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qg.dat qr_acr_qg.dat
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qs.dat qr_acr_qs.dat
+  cp    @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN CCN_ACTIVATE.BIN
+  cp    @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat freezeH2O.dat
+  cp    @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat qr_acr_qgV2.dat
+  cp    @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat qr_acr_qsV2.dat
 fi
 
 #prognostic aerosols

diff --git a/tests/fv3_conf/hafs_fv3_run.IN b/tests/fv3_conf/hafs_fv3_run.IN
@@ -30,10 +30,10 @@ else
 fi
 
 if [ ${CNTL_DIR} = "hafs_regional_atm_thompson_gfdlsf" ]; then
-  cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .
-  cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qgV2.dat .
-  cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
-  cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
+  cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat .
+  cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat .
+  cp @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat .
+  cp @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN .
 fi
 
 if [[ $POSTAPP = 'hafs' ]]; then

diff --git a/tests/fv3_conf/regional_run.IN b/tests/fv3_conf/regional_run.IN
@@ -25,13 +25,13 @@ cp @[INPUTDATA_ROOT]/FV3_input_data_INCCN_aeroclim/aer_data/LUTS/optics_SS.v3_3.
 cp @[INPUTDATA_ROOT]/FV3_input_data_INCCN_aeroclim/aer_data/LUTS/optics_SU.v1_3.dat  optics_SU.dat
 
 if [ $WARM_START = .true. ]; then
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.coupler.res             INPUT/coupler.res
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.nc          INPUT/fv_core.res.nc
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.tile1.nc    INPUT/fv_core.res.tile1.nc
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_srf_wnd.res.tile1.nc INPUT/fv_srf_wnd.res.tile1.nc
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_tracer.res.tile1.nc  INPUT/fv_tracer.res.tile1.nc
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.phy_data.nc             INPUT/phy_data.nc
-    cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.sfc_data.nc             INPUT/sfc_data.nc
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.coupler.res             INPUT/coupler.res
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.nc          INPUT/fv_core.res.nc
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.tile1.nc    INPUT/fv_core.res.tile1.nc
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_srf_wnd.res.tile1.nc INPUT/fv_srf_wnd.res.tile1.nc
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_tracer.res.tile1.nc  INPUT/fv_tracer.res.tile1.nc
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.phy_data.nc             INPUT/phy_data.nc
+    cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.sfc_data.nc             INPUT/sfc_data.nc
 fi
 if [ $WRITE_RESTART_WITH_BCS = .true. ]; then
     cp @[INPUTDATA_ROOT]/FV3_regional/RESTART/fv_core.res.tile1_new.nc                           RESTART/fv_core.res.tile1_new.nc

diff --git a/tests/fv3_conf/rrfs_warm_run.IN b/tests/fv3_conf/rrfs_warm_run.IN
@@ -43,10 +43,10 @@ touch data_table
 
 # Thompson MP lookup tables - copy standard and SIONlib tables of precomputed tables
 if [ $IMP_PHYSICS = 8 ]; then
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qgV2.dat .
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
-  cp    @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat .
+  cp    @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN .
 fi
 
 cp ${PATHRT}/parm/postxconfig-NT-fv3lam.txt postxconfig-NT.txt

diff --git a/tests/parm/model_configure.IN b/tests/parm/model_configure.IN
@@ -12,6 +12,7 @@ restart_interval:        @[RESTART_INTERVAL]
 output_1st_tstep_rst:    .false.
 
 quilting:                @[QUILTING]
+quilting_restart:        @[QUILTING_RESTART]
 write_groups:            @[WRITE_GROUP]
 write_tasks_per_group:   @[WRTTASK_PER_GROUP]
 itasks:                  @[ITASKS]

diff --git a/tests/parm/model_configure_regional.IN b/tests/parm/model_configure_regional.IN
@@ -13,6 +13,7 @@ cpl:                     .false.
 write_dopost:            @[WRITE_DOPOST]
 output_history:          .true.
 quilting:                @[QUILTING]
+quilting_restat:         @[QUILTING_RESTART]
 write_groups:            1
 write_tasks_per_group:   10
 num_files:               2

diff --git a/tests/rt.conf b/tests/rt.conf
@@ -8,14 +8,16 @@ COMPILE | -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgsho
 RUN     | cpld_control_gfsv17                                                                                                     | - wcoss2.intel                          | fv3 |
 RUN     | cpld_control_p8                                                                                                         |                                         | fv3 |
 RUN     | cpld_restart_p8                                                                                                         |                                         |     | cpld_control_p8
+RUN     | cpld_control_qr_p8                                                                                                      |                                         |     |
+RUN     | cpld_restart_qr_p8                                                                                                      |                                         |     | cpld_control_qr_p8
 RUN     | cpld_2threads_p8                                                                                                        |                                         |     |
 RUN     | cpld_decomp_p8                                                                                                          |                                         |     |
 RUN     | cpld_mpi_p8                                                                                                             |                                         |     |
 #RUN     | cpld_multigrid_p8                                                                                                       |                                         |     |
-RUN     | cpld_control_ciceC_p8                                                                                                   |                                         | fv3 |
+RUN     | cpld_control_ciceC_p8                                                                                                   | - gaea.intel                            | fv3 |
 
-RUN     | cpld_control_c192_p8                                                                                                    | - wcoss2.intel jet.intel acorn.intel    | fv3 |
-RUN     | cpld_restart_c192_p8                                                                                                    | - wcoss2.intel jet.intel acorn.intel    |     | cpld_control_c192_p8
+RUN     | cpld_control_c192_p8                                                                                                    | - wcoss2.intel jet.intel acorn.intel gaea.intel   | fv3 |
+RUN     | cpld_restart_c192_p8                                                                                                    | - wcoss2.intel jet.intel acorn.intel gaea.intel   |     | cpld_control_c192_p8
 
 RUN     | cpld_bmark_p8                                                                                                           | - jet.intel cheyenne.intel acorn.intel  | fv3 |
 RUN     | cpld_restart_bmark_p8                                                                                                   | - jet.intel cheyenne.intel acorn.intel  |     | cpld_bmark_p8
@@ -62,10 +64,12 @@ RUN     | control_iovr4
 RUN     | control_iovr5                                                                                                           |                                         | fv3 |
 
 RUN     | control_p8                                                                                                              |                                         | fv3 |
-RUN     | control_p8_lndp                                                                                                         |                                         | fv3 |
 RUN     | control_restart_p8                                                                                                      |                                         |     | control_p8
+RUN     | control_qr_p8                                                                                                           |                                         |     |
+RUN     | control_restart_qr_p8                                                                                                   |                                         |     | control_qr_p8
 RUN     | control_decomp_p8                                                                                                       |                                         |     |
 RUN     | control_2threads_p8                                                                                                     |                                         |     |
+RUN     | control_p8_lndp                                                                                                         |                                         | fv3 |
 
 RUN     | control_p8_rrtmgp                                                                                                       |                                         | fv3 |
 RUN     | control_p8_mynn                                                                                                         |                                         | fv3 |
@@ -74,6 +78,8 @@ RUN     | merra2_thompson
 
 RUN     | regional_control                                                                                                        |                                         | fv3 |
 RUN     | regional_restart                                                                                                        |                                         |     | regional_control
+RUN     | regional_control_qr                                                                                                     |                                         |     |
+RUN     | regional_restart_qr                                                                                                     |                                         |     | regional_control_qr
 RUN     | regional_decomp                                                                                                         |                                         |     |
 RUN     | regional_2threads                                                                                                       |                                         |     |
 RUN     | regional_noquilt                                                                                                        |                                         | fv3 |
@@ -113,6 +119,9 @@ RUN     | rrfs_smoke_conus13km_radar_tten_warm
 #RUN     | rrfs_smoke_conus13km_hrrr_warm_decomp                                                                                   |                                         |     |
 #RUN     | rrfs_smoke_conus13km_hrrr_warm_restart                                                                                   |                                        |     | rrfs_smoke_conus13km_hrrr_warm
 
+# Just to make sure restart doesn't crash again:
+RUN     | rrfs_conus13km_hrrr_warm_restart_mismatch                                                                               |                                         | fv3 | rrfs_conus13km_hrrr_warm
+
 COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_csawmg,FV3_GFS_v16_ugwpv1,FV3_GFS_v16_ras,FV3_GFS_v16_noahmp                        |                                         | fv3 |
 RUN     | control_csawmg                                                                                                          | - gaea.intel                            | fv3 |
 RUN     | control_csawmgt                                                                                                         |                                         | fv3 |

diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh
@@ -446,7 +446,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20230413
+BL_DATE=20230418
 
 RTPWD=${RTPWD:-$DISKNM/NCAR/main-${BL_DATE}/${RT_COMPILER^^}}
 

diff --git a/tests/rt.sh b/tests/rt.sh
@@ -446,7 +446,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
 fi
 
 
-BL_DATE=20230324
+BL_DATE=20230329
 
 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 

diff --git a/tests/rt_gnu.conf b/tests/rt_gnu.conf
@@ -33,6 +33,9 @@ RUN     | rrfs_smoke_conus13km_radar_tten_warm
 #RUN     | rrfs_smoke_conus13km_hrrr_warm_decomp                                                                                   |                                         |     |
 #RUN     | rrfs_smoke_conus13km_hrrr_warm_restart                                                                                   |                                        |     | rrfs_smoke_conus13km_hrrr_warm
 
+# Just to make sure restart doesn't crash again:
+RUN     | rrfs_conus13km_hrrr_warm_restart_mismatch                                                                               |                                         | fv3 | rrfs_conus13km_hrrr_warm
+
 ##################################################################################################################################################################
 # CCPP DEBUG tests                                                                                                                                               #
 ##################################################################################################################################################################

diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh
@@ -38,8 +38,8 @@ function compute_petbounds_and_tasks() {
   # ATM
   ATM_io_tasks=${ATM_io_tasks:-0}
   if [[ $((ATM_compute_tasks + ATM_io_tasks)) -gt 0 ]]; then
-     atm_petlist_bounds="${n} $((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks - 1))"
-     n=$((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks))
+     atm_petlist_bounds="${n} $((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks*atm_omp_num_threads - 1))"
+     n=$((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks*atm_omp_num_threads))
   fi
 
   # OCN

diff --git a/tests/run_test.sh b/tests/run_test.sh
@@ -100,6 +100,11 @@ cp ${PATHTR}/modulefiles/ufs_common*               ./modulefiles/.
 # Get the shell file that loads the "module" command and purges modules:
 cp ${PATHRT}/module-setup.sh                       module-setup.sh
 
+if [[ $MACHINE_ID == wcoss2.* ]] || [[ $MACHINE_ID == acorn.* ]] ; then
+  # for compare_ncfile.py
+  module load gcc/10.3.0 python/3.8.6
+fi
+
 SRCD="${PATHTR}"
 RUND="${RUNDIR}"
+1 −1		cicecore/cicedyn/analysis/ice_history.F90
+6 −5		cicecore/cicedyn/dynamics/ice_dyn_shared.F90
+3 −1		cicecore/cicedyn/general/ice_flux.F90
+1 −1		cicecore/cicedyn/general/ice_init.F90
+4 −2		cicecore/cicedyn/general/ice_step_mod.F90
+287 −28		cicecore/cicedyn/infrastructure/comm/mpi/ice_boundary.F90
+338 −104		cicecore/cicedyn/infrastructure/comm/serial/ice_boundary.F90
+5 −1		cicecore/cicedyn/infrastructure/ice_grid.F90
+9 −8		cicecore/cicedyn/infrastructure/io/io_binary/ice_history_write.F90
+28 −7		cicecore/cicedyn/infrastructure/io/io_netcdf/ice_history_write.F90
+25 −6		cicecore/cicedyn/infrastructure/io/io_pio2/ice_history_write.F90
+6 −0		cicecore/cicedyn/infrastructure/io/io_pio2/ice_restart.F90
+2 −2		cicecore/drivers/direct/nemo_concepts/CICE_InitMod.F90
+2 −2		cicecore/drivers/standalone/cice/CICE_InitMod.F90
+472 −0		cicecore/drivers/unittest/halochk/CICE_InitMod.F90
+811 −0		cicecore/drivers/unittest/halochk/halochk.F90
+1 −2		cicecore/shared/ice_constants.F90
+4 −2		configuration/scripts/Makefile
+4 −2		configuration/scripts/cice_decomp.csh
+2 −0		configuration/scripts/options/set_env.halochk
+3 −0		configuration/scripts/options/set_nml.cyclic
+3 −0		configuration/scripts/options/set_nml.open
+3 −0		configuration/scripts/options/set_nml.tripole
+3 −0		configuration/scripts/options/set_nml.tripolet
+28 −13		configuration/scripts/tests/unittest_suite.ts
+4 −3		doc/source/cice_index.rst
+3 −0		doc/source/conf.py
+10 −0		doc/source/master_list.bib
+17 −9		doc/source/science_guide/sg_dynamics.rst
+1 −1		doc/source/user_guide/ug_case_settings.rst
+53 −10		doc/source/user_guide/ug_implementation.rst
+1 −0		doc/source/user_guide/ug_testing.rst
+1 −1		icepack
+15 −2		ufs/ccpp/data/MED_typedefs.F90
+47 −0		ufs/ccpp/data/MED_typedefs.meta
+2 −0		CMakeLists.txt
+1 −1		atmos_cubed_sphere
+22 −18		atmos_model.F90
+0 −28		ccpp/data/CCPP_typedefs.meta
+111 −14		ccpp/data/GFS_typedefs.F90
+130 −11		ccpp/data/GFS_typedefs.meta
+166 −16		ccpp/driver/GFS_diagnostics.F90
+1 −1		ccpp/physics
+89 −42		fv3_cap.F90
+114 −76		io/FV3GFS_io.F90
+920 −0		io/FV3GFS_restart_io.F90
+0 −3		io/module_write_internal_state.F90
+31 −19		io/module_write_netcdf.F90
+587 −0		io/module_write_restart_netcdf.F90
+392 −118		io/module_wrt_grid_comp.F90
+19 −4		io/post_fv3.F90
+180 −27		module_fcst_grid_comp.F90
+1 −1		module_fv3_config.F90
-Original file line number
+Diff line change
@@ Expand Up @@
     fi
-    BL_DATE=20230324
+    BL_DATE=20230329
     RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
@@ Expand Down @@