Multiple output (nested) grids from fv3atm (#1044)

Update fv3atm to support writing multiple history files for parent and nested domains. Nested HAFS regression tests were updated to write out history files from all nested domains. Co-authored-by: Bin Liu <[email protected]>
ufs-community · Feb 15, 2022 · dce9936 · dce9936
1 parent cad761d
commit dce9936
Show file tree

Hide file tree

Showing 26 changed files with 5,861 additions and 6,343 deletions.
diff --git a/FV3 b/FV3
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log
diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -1442,6 +1442,7 @@ export DLAT=0.03
 
 # input.nml
 export CPL_IMP_MRG=.true.
+
 export DIAG_TABLE=diag_table_hafs
 export FIELD_TABLE=field_table_hafs
 
@@ -1453,3 +1454,131 @@ export CPLMODE=hafs
 export RUNTYPE=startup
 export USE_COLDSTART=false
 }
+
+export_hafs ()
+{
+export FV3=true
+export S2S=false
+export HAFS=true
+export DATM_CDEPS=false
+export DOCN_CDEPS=false
+export THRD=1
+export INPES=$INPES_dflt
+export JNPES=$JNPES_dflt
+export TASKS=$TASKS_dflt
+export TPN=$TPN_dflt
+
+export OUTPUT_GRID=''
+export IMO=''
+export JMO=''
+export CEN_LON=''
+export CEN_LAT=''
+export LON1=''
+export LAT1=''
+export LON2=''
+export LAT2=''
+export DLON=''
+export DLAT=''
+export STDLAT1=''
+export STDLAT2=''
+export NX=''
+export NY=''
+export DX=''
+export DY=''
+
+export OUTPUT_GRID_2=''
+export IMO_2=''
+export JMO_2=''
+export CEN_LON_2=''
+export CEN_LAT_2=''
+export LON1_2=''
+export LAT1_2=''
+export LON2_2=''
+export LAT2_2=''
+export DLON_2=''
+export DLAT_2=''
+export STDLAT1_2=''
+export STDLAT2_2=''
+export NX_2=''
+export NY_2=''
+export DX_2=''
+export DY_2=''
+
+export OUTPUT_GRID_3=''
+export IMO_3=''
+export JMO_3=''
+export CEN_LON_3=''
+export CEN_LAT_3=''
+export LON1_3=''
+export LAT1_3=''
+export LON2_3=''
+export LAT2_3=''
+export DLON_3=''
+export DLAT_3=''
+export STDLAT1_3=''
+export STDLAT2_3=''
+export NX_3=''
+export NY_3=''
+export DX_3=''
+export DY_3=''
+
+export OUTPUT_GRID_4=''
+export IMO_4=''
+export JMO_4=''
+export CEN_LON_4=''
+export CEN_LAT_4=''
+export LON1_4=''
+export LAT1_4=''
+export LON2_4=''
+export LAT2_4=''
+export DLON_4=''
+export DLAT_4=''
+export STDLAT1_4=''
+export STDLAT2_4=''
+export NX_4=''
+export NY_4=''
+export DX_4=''
+export DY_4=''
+
+export OUTPUT_GRID_5=''
+export IMO_5=''
+export JMO_5=''
+export CEN_LON_5=''
+export CEN_LAT_5=''
+export LON1_5=''
+export LAT1_5=''
+export LON2_5=''
+export LAT2_5=''
+export DLON_5=''
+export DLAT_5=''
+export STDLAT1_5=''
+export STDLAT2_5=''
+export NX_5=''
+export NY_5=''
+export DX_5=''
+export DY_5=''
+
+export OUTPUT_GRID_6=''
+export IMO_6=''
+export JMO_6=''
+export CEN_LON_6=''
+export CEN_LAT_6=''
+export LON1_6=''
+export LAT1_6=''
+export LON2_6=''
+export LAT2_6=''
+export DLON_6=''
+export DLAT_6=''
+export STDLAT1_6=''
+export STDLAT2_6=''
+export NX_6=''
+export NY_6=''
+export DX_6=''
+export DY_6=''
+
+export NFHOUT=3
+export NFHMAX_HF=-1
+export NFHOUT_HF=3
+export NSOUT=-1
+export OUTPUT_FH=-1
+}
diff --git a/tests/parm/model_configure_hafs_shared.IN b/tests/parm/model_configure_hafs_shared.IN
@@ -25,19 +25,126 @@ kchunk3d:                -1
 ideflate:                @[IDEFLATE]
 nbits:                   @[NBITS]
 
-app_domain:              'regional'
-output_grid:             @[OUTPUT_GRID]
-cen_lon:                 @[CEN_LON]             # central longitude
-cen_lat:                 @[CEN_LAT]             # central latitude
-lon1:                    @[LON1]                # longitude of lower-left
-lat1:                    @[LAT1]                # latitude of lower-left
-lon2:                    @[LON2]                # longitude of upper-right
-lat2:                    @[LAT2]                # latitude of upper-right
-dlon:                    @[DLON]
-dlat:                    @[DLAT]
+output_grid:             @[OUTPUT_GRID]     # gaussian_grid, global_latlon, regional_latlon, rotated_latlon, lambert_conformal, cubed_sphere_grid
+imo:                     @[IMO]             # number of grid points along longitude/latitude direction
+jmo:                     @[JMO]             # needed for gaussian_grid or global_latlon
+cen_lon:                 @[CEN_LON]         # central longitude/latitude (degrees)
+cen_lat:                 @[CEN_LAT]         # needed for rotated_latlon, lambert_conformal
+lon1:                    @[LON1]            # longitude/latitude of lower-left corner
+lat1:                    @[LAT1]            # needed for regional_latlon, rotated_latlon, lambert_conformal
+lon2:                    @[LON2]            # longitude/latitude of upper-right corner
+lat2:                    @[LAT2]            # needed for regional_latlon, rotated_latlon
+dlon:                    @[DLON]            # grid spacing in longitude/latitude direction (degrees)
+dlat:                    @[DLAT]            # needed for regional_latlon, rotated_latlon
+stdlat1:                 @[STDLAT1]         # latitude of first/second standard parallel (degrees)
+stdlat2:                 @[STDLAT2]         # needed for lambert_conformal
+nx:                      @[NX]              # number of grid cells along x/y-axis
+ny:                      @[NY]              # needed for lambert_conformal
+dx:                      @[DX]              # grid cell size in x/y direction (meters)
+dy:                      @[DY]              # needed for lambert_conformal
 
-nfhout:                  @[NFHOUT]
-nfhmax_hf:               @[NFHMAX_HF]
-nfhout_hf:               @[NFHOUT_HF]
-nsout:                   -1
-output_fh:               -1
+<output_grid_02>
+output_grid:             @[OUTPUT_GRID_2]
+imo:                     @[IMO_2]
+jmo:                     @[JMO_2]
+cen_lon:                 @[CEN_LON_2]
+cen_lat:                 @[CEN_LAT_2]
+lon1:                    @[LON1_2]
+lat1:                    @[LAT1_2]
+lon2:                    @[LON2_2]
+lat2:                    @[LAT2_2]
+dlon:                    @[DLON_2]
+dlat:                    @[DLAT_2]
+stdlat1:                 @[STDLAT1_2]
+stdlat2:                 @[STDLAT2_2]
+nx:                      @[NX_2]
+ny:                      @[NY_2]
+dx:                      @[DX_2]
+dy:                      @[DY_2]
+</output_grid_02>
+<output_grid_03>
+output_grid:             @[OUTPUT_GRID_3]
+imo:                     @[IMO_3]
+jmo:                     @[JMO_3]
+cen_lon:                 @[CEN_LON_3]
+cen_lat:                 @[CEN_LAT_3]
+lon1:                    @[LON1_3]
+lat1:                    @[LAT1_3]
+lon2:                    @[LON2_3]
+lat2:                    @[LAT2_3]
+dlon:                    @[DLON_3]
+dlat:                    @[DLAT_3]
+stdlat1:                 @[STDLAT1_3]
+stdlat2:                 @[STDLAT2_3]
+nx:                      @[NX_3]
+ny:                      @[NY_3]
+dx:                      @[DX_3]
+dy:                      @[DY_3]
+</output_grid_03>
+<output_grid_04>
+output_grid:             @[OUTPUT_GRID_4]
+imo:                     @[IMO_4]
+jmo:                     @[JMO_4]
+cen_lon:                 @[CEN_LON_4]
+cen_lat:                 @[CEN_LAT_4]
+lon1:                    @[LON1_4]
+lat1:                    @[LAT1_4]
+lon2:                    @[LON2_4]
+lat2:                    @[LAT2_4]
+dlon:                    @[DLON_4]
+dlat:                    @[DLAT_4]
+stdlat1:                 @[STDLAT1_4]
+stdlat2:                 @[STDLAT2_4]
+nx:                      @[NX_4]
+ny:                      @[NY_4]
+dx:                      @[DX_4]
+dy:                      @[DY_4]
+</output_grid_04>
+<output_grid_05>
+output_grid:             @[OUTPUT_GRID_5]
+imo:                     @[IMO_5]
+jmo:                     @[JMO_5]
+cen_lon:                 @[CEN_LON_5]
+cen_lat:                 @[CEN_LAT_5]
+lon1:                    @[LON1_5]
+lat1:                    @[LAT1_5]
+lon2:                    @[LON2_5]
+lat2:                    @[LAT2_5]
+dlon:                    @[DLON_5]
+dlat:                    @[DLAT_5]
+stdlat1:                 @[STDLAT1_5]
+stdlat2:                 @[STDLAT2_5]
+nx:                      @[NX_5]
+ny:                      @[NY_5]
+dx:                      @[DX_5]
+dy:                      @[DY_5]
+</output_grid_05>
+<output_grid_06>
+output_grid:             @[OUTPUT_GRID_6]
+imo:                     @[IMO_6]
+jmo:                     @[JMO_6]
+cen_lon:                 @[CEN_LON_6]
+cen_lat:                 @[CEN_LAT_6]
+lon1:                    @[LON1_6]
+lat1:                    @[LAT1_6]
+lon2:                    @[LON2_6]
+lat2:                    @[LAT2_6]
+dlon:                    @[DLON_6]
+dlat:                    @[DLAT_6]
+stdlat1:                 @[STDLAT1_6]
+stdlat2:                 @[STDLAT2_6]
+nx:                      @[NX_6]
+ny:                      @[NY_6]
+dx:                      @[DX_6]
+dy:                      @[DY_6]
+</output_grid_06>
+
+nfhout:                  @[NFHOUT]          # Output frequency in hours after forecast hour nfhmax_hf
+nfhmax_hf:               @[NFHMAX_HF]       # Number of forecast hours until output frequency nfhout takes affect
+nfhout_hf:               @[NFHOUT_HF]       # Output frequency in hours until forecast hour nfhmax_hf
+nsout:                   @[NSOUT]           # Output frequency in time steps (positive values override nfhout and nfhout_hf)
+output_fh:               @[OUTPUT_FH]       # Output forecast hours controlled by output_fh (see notes below)
+# a) An array of output_fh means to output history files at those forecast time (e.g., "output_fh: 0 1 2 3 12 24 48")
+#    If the first elelment is zero, it means the first time step output
+# b) If there are two elements in output_fh and the second one is -1, then the first element is output frequency. e.g.
+#    For example, "output_fh: 6 -1" will output the history file every 6 hours.
diff --git a/tests/rt.sh b/tests/rt.sh
@@ -479,7 +479,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20220207
+BL_DATE=20220214
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]] || [[ $MACHINE_ID = s4.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else

diff --git a/tests/run_test.sh b/tests/run_test.sh
@@ -114,27 +114,44 @@ if [[ "Q${INPUT_NEST02_NML:-}" != Q ]] ; then
     NPX_NEST=$NPX_NEST02; NPY_NEST=$NPY_NEST02
     K_SPLIT_NEST=$K_SPLIT_NEST02; N_SPLIT_NEST=$N_SPLIT_NEST02
     atparse < ${PATHRT}/parm/${INPUT_NEST02_NML} > input_nest02.nml
+else
+    sed -i -e "/<output_grid_02>/,/<\/output_grid_02>/d" model_configure
 fi
 
 if [[ "Q${INPUT_NEST03_NML:-}" != Q ]] ; then
     INPES_NEST=$INPES_NEST03; JNPES_NEST=$JNPES_NEST03
     NPX_NEST=$NPX_NEST03; NPY_NEST=$NPY_NEST03
     K_SPLIT_NEST=$K_SPLIT_NEST03; N_SPLIT_NEST=$N_SPLIT_NEST03
     atparse < ${PATHRT}/parm/${INPUT_NEST03_NML} > input_nest03.nml
+else
+    sed -i -e "/<output_grid_03>/,/<\/output_grid_03>/d" model_configure
 fi
 
 if [[ "Q${INPUT_NEST04_NML:-}" != Q ]] ; then
     INPES_NEST=$INPES_NEST04; JNPES_NEST=$JNPES_NEST04
     NPX_NEST=$NPX_NEST04; NPY_NEST=$NPY_NEST04
     K_SPLIT_NEST=$K_SPLIT_NEST04; N_SPLIT_NEST=$N_SPLIT_NEST04
     atparse < ${PATHRT}/parm/${INPUT_NEST04_NML} > input_nest04.nml
+else
+    sed -i -e "/<output_grid_04>/,/<\/output_grid_04>/d" model_configure
 fi
 
 if [[ "Q${INPUT_NEST05_NML:-}" != Q ]] ; then
     INPES_NEST=$INPES_NEST05; JNPES_NEST=$JNPES_NEST05
     NPX_NEST=$NPX_NEST05; NPY_NEST=$NPY_NEST05
     K_SPLIT_NEST=$K_SPLIT_NEST05; N_SPLIT_NEST=$N_SPLIT_NEST05
     atparse < ${PATHRT}/parm/${INPUT_NEST05_NML} > input_nest05.nml
+else
+    sed -i -e "/<output_grid_05>/,/<\/output_grid_05>/d" model_configure
+fi
+
+if [[ "Q${INPUT_NEST06_NML:-}" != Q ]] ; then
+    INPES_NEST=$INPES_NEST06; JNPES_NEST=$JNPES_NEST06
+    NPX_NEST=$NPX_NEST06; NPY_NEST=$NPY_NEST06
+    K_SPLIT_NEST=$K_SPLIT_NEST06; N_SPLIT_NEST=$N_SPLIT_NEST06
+    atparse < ${PATHRT}/parm/${INPUT_NEST06_NML} > input_nest06.nml
+else
+    sed -i -e "/<output_grid_06>/,/<\/output_grid_06>/d" model_configure
 fi
 
 # diag table

diff --git a/tests/tests/hafs_global_1nest_atm b/tests/tests/hafs_global_1nest_atm
@@ -9,37 +9,45 @@ export TEST_DESCR="Compare HAFS global with 1 nest and atmosphere only results w
 export CNTL_DIR=hafs_global_1nest_atm
 
 export LIST_FILES="atmf006.nc \
-                   sfcf006.nc"
+                   sfcf006.nc \
+                   atm.nest02.f006.nc \
+                   sfc.nest02.f006.nc"
 
 export_fv3
+export_hafs
 
 export HAFS=true
 export FHMAX=6
 export RESTART_N=${FHMAX}
 export TASKS=240
 export THRD=2
 export TPN=$TPN_thrd
-export CEN_LON=-86.0
-export CEN_LAT=23.0
-export LON1=-45.00
-export LAT1=-40.00
-export LON2=45.00
-export LAT2=40.00
-export DLON=0.1
-export DLAT=0.1
 export DT_ATMOS=90
 export IDEFLATE=1
 export NFHMAX_HF=-1
 export NFHOUT_HF=3
 export NFHOUT=3
-export OUTPUT_FILE="'netcdf_parallel' 'netcdf_parallel'"
-export OUTPUT_GRID='rotated_latlon'
+export OUTPUT_FILE="'netcdf_parallel' 'netcdf'"
 export SDAY=25
 export SHOUR=12
 export SMONTH=08
 export SYEAR=2020
 export WRTTASK_PER_GROUP=60
 
+export OUTPUT_GRID='global_latlon'
+export IMO=360
+export JMO=181
+
+export OUTPUT_GRID_2='rotated_latlon'
+export CEN_LON_2=-86.0
+export CEN_LAT_2=23.0
+export LON1_2=-45.00
+export LAT1_2=-40.00
+export LON2_2=45.00
+export LAT2_2=40.00
+export DLON_2=0.25
+export DLAT_2=0.25
+
 export BLOCKSIZE=${BLOCKSIZE:-24}
 export CCPP_SUITE="FV3_HAFS_v0_gfdlmp_tedmf"
 export DEFLATE_LEVEL=-1
+1 −1		atmos_cubed_sphere
+14 −3		atmos_model.F90
+15 −43		fv3_cap.F90
+7 −2		io/FV3GFS_io.F90
+16 −14		io/inline_post.F90
+14 −12		io/module_fv3_io_def.F90
+50 −49		io/module_write_netcdf.F90
+524 −592		io/module_wrt_grid_comp.F90
+44 −43		io/post_regional.F90
+348 −174		module_fcst_grid_comp.F90