GFSv16 netcdf post ficein cpl (#36)

* Updated fv3atm (see fv3atm PR #48 NOAA-EMC/fv3atm#48) * Updated WW3
ufs-community · Jan 25, 2020 · 52795b8 · 52795b8
1 parent b41e023
commit 52795b8
Show file tree

Hide file tree

Showing 55 changed files with 12,112 additions and 125,625 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+coupled*.mk
diff --git a/FV3 b/FV3
diff --git a/WW3 b/WW3
diff --git a/cmake/configure_gaea.cmake b/cmake/configure_gaea.cmake
@@ -17,6 +17,8 @@ option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
 
+option(INLINE_POST "Enable inline post" OFF)
+
 include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
 
 set(NEMSIO_INC $ENV{NEMSIO_INC})

diff --git a/compsets/all.input b/compsets/all.input
@@ -3,7 +3,7 @@ load 'fv3.input'
 run fv3_control                 @ fv3, standard, baseline, ccpptrans
 run fv3_decomp                  @ fv3, standard
 run fv3_2threads                @ fv3, standard
-run fv3_ccpp_control            @ fv3, standard, ccpptrans, plat==hera.intel
+#run fv3_ccpp_control            @ fv3, standard, ccpptrans, plat==hera.intel
 run fv3_restart                 @ fv3, standard, baseline
 run fv3_read_inc                @ fv3, standard, baseline
 run fv3_gfdlmp                  @ fv3, standard, baseline
@@ -22,7 +22,7 @@ run fv3_wrtGauss_netcdf_esmf    @ fv3, standard, baseline
 run fv3_wrtGauss_netcdf         @ fv3, standard, baseline
 run fv3_wrtGauss_nemsio         @ fv3, standard, baseline
 
-run fv3_wrtGauss_nemsio_c768    @ fv3,           baseline, ww3, plat==hera.intel, plat==wcoss.cray
+run fv3_wrtGauss_nemsio_c768    @ fv3,           baseline, ww3, plat==hera.intel
 run fv3_wrtGauss_nemsio_c192    @ fv3, standard, baseline
 run fv3_stochy                  @ fv3, standard, baseline
 run fv3_iau                     @ fv3, standard, baseline

diff --git a/compsets/fv3.input b/compsets/fv3.input
@@ -1764,10 +1764,11 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
     TEST_DESCR="Compare c768 FV3-WW3 Gaussian grid nemsio output results with previous trunk version"
     CNTL_NAME="fv3_wrtGauss_nemsio_c768"
 
-    TASKS='1752'
+    TASKS='1470'
     PPN="@[plat%C768_PPN]"
+    THRD="@[plat%C768_THRD]"
     INPES='16'
-    JNPES='16'
+    JNPES='12'
     NPX='769'
     NPY='769'
     NPZ='64'
@@ -1779,7 +1780,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
     FHMAX='06'
     WLCLK='30'
     WRITE_GROUP='3'
-    WRTTASK_PER_GROUP='64'
+    WRTTASK_PER_GROUP='36'
     FDIAG='0,1,2,3,4,5,6,7,8,9,10,11,12'
     OUTPUT_GRID="'gaussian_grid'"
     OUTPUT_FILE="'nemsio'"
@@ -1805,9 +1806,9 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
 
     nems_configure="blocked_atm_wav"
     atm_model='fv3'
-    atm_petlist_bounds="0 1727"
+    atm_petlist_bounds="0 1259"
     wav_model='ww3'
-    wav_petlist_bounds="1728 1751"
+    wav_petlist_bounds="1260 1469"
     coupling_interval_sec=1800.0  # 30min coupling, fv3 time step
 
     # The build variable is the fv3_ww3.exe, which
@@ -1834,6 +1835,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
      'nems.configure' <=atparse=   "@[PARMnems]/nems.configure.@[nems_configure].IN"
           'mod_def.*' <=copyfrom=  "@[WW3_input_data]"
       'ww3_multi.inp' <=copyfrom=  "@[WW3_input_data]/@[SYEAR]@[SMONTH]@[SDAY]"
+      'rmp_src*' <=copyfrom=  "@[WW3_input_data]/@[SYEAR]@[SMONTH]@[SDAY]"
     }
 
     # Specify output files:
@@ -1879,7 +1881,9 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
         'RESTART/sfc_data.tile4.nc'             .bitcmp. "@[CNTL]/RESTART/"
         'RESTART/sfc_data.tile5.nc'             .bitcmp. "@[CNTL]/RESTART/"
         'RESTART/sfc_data.tile6.nc'             .bitcmp. "@[CNTL]/RESTART/"
-        'out_grd.glo_30m'                       .bitcmp. "@[CNTL]"
+        'out_grd.glo_10m'                       .bitcmp. "@[CNTL]"
+        'out_grd.ant_9km'                       .bitcmp. "@[CNTL]"
+        'out_grd.aoc_9km'                       .bitcmp. "@[CNTL]"
       # Executable validation.  This makes an MD5 sum of the fv3.exe
       # for comparison against the MD5 sum made in the build job.
       # This is to ensure the executable did not change during the
@@ -1888,7 +1892,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
     }
 
     spawn execute {
-        {"@[build%target]", ranks="@[TASKS]", ppn="@[PPN]" }
+        {"@[build%target]", ranks="@[TASKS]", threads="@[THRD]", ppn="@[PPN]" }
     }
 }
 

diff --git a/compsets/hera.input b/compsets/hera.input
@@ -16,7 +16,8 @@ platform hera.intel {
 
     cores_per_node=40 # Number of cores per node on compute nodes
     cpus_per_core=2
-    C768_PPN=30
+    C768_PPN=10
+    C768_THRD=1
 
  # hera testing: using longer run time
     DEFAULT_TEST_WALLTIME=3000
@@ -30,8 +31,8 @@ platform hera.intel {
     # INPUTS is the input directory, which should contain fix and parm
     # files, plus any restarts or other inputs.
 
-    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
-    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
+    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
     INPUTS="@[BASELINE]"
 
     default_resources={

diff --git a/compsets/platforms.input b/compsets/platforms.input
@@ -26,6 +26,7 @@ platform.defaults={
     walltime="@[DEFAULT_TEST_WALLTIME]"
 
     C768_PPN="@[cores_per_node]"
+    C768_THRD=1
     execution_time_modules=""
 }
 

diff --git a/compsets/wcoss.input b/compsets/wcoss.input
@@ -17,6 +17,8 @@ platform wcoss.phase2 {
     pex='1'
     cores_per_node=24
     cpus_per_core=2
+    C768_PPN=6
+    C768_THRD=1
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=3000
@@ -26,8 +28,8 @@ platform wcoss.phase2 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
     MACHINE_ID='wcoss.phase2'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
 
     default_resources={
         TASKS=156
@@ -120,9 +122,12 @@ platform wcoss.phase1 {
     pex='1'
     cores_per_node=16
     cpus_per_core=2
+    C768_PPN=6
+    C768_THRD=1
+
     MACHINE_ID='wcoss.phase1'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=2700
@@ -197,25 +202,27 @@ platform wcoss_dell_p3 {
     pex='3'
     cores_per_node=28
     cpus_per_core=2
+    C768_PPN=7
+    C768_THRD=1
 
-    DEFAULT_TEST_WALLTIME=1800
+    DEFAULT_TEST_WALLTIME=3000
 
     MPI='LSF_IMPI'
     MACHINE_ID='wcoss_dell_p3'
     SHORT_TEST_QUEUE='&SHORTQ;'
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
 
-    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
-    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
+    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
+    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
 
     default_resources={
         TASKS=156
         WRTTASK_PER_GROUP=12
-        TPN=8
+        TPN=6
         INPES=3
         JNPES=8
-        THRD=1
+        THRD=4
         atm_petlist_bounds="0 155"
         wav_petlist_bounds="156 263"
         CPLTASKS=264
@@ -295,6 +302,9 @@ platform wcoss.cray {
     pex='cray'
     cores_per_node=24
     cpus_per_core=2
+    C768_PPN=6
+    C768_THRD=1
+
     MPI='lsf_cray_intel'
     SHORT_TEST_QUEUE='&SHORTQ;'
     LONG_TEST_QUEUE='&LONGQ;'
@@ -304,8 +314,8 @@ platform wcoss.cray {
     BUILD_WALLTIME="3600"
     DEFAULT_TEST_WALLTIME=1800
 
-    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
-    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20191230"
+    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
 
     execution_time_modules=[[[
       module load alps

diff --git a/conf/configure.fv3.gaea.intel b/conf/configure.fv3.gaea.intel
@@ -66,6 +66,10 @@ else
 CPPDEFS += -DMOIST_CAPPA -DUSE_COND
 endif
 
+ifeq ($(NAM_phys),Y)
+CPPDEFS += -DNAM_phys
+endif
+
 ifeq ($(32BIT),Y)
 CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
 FFLAGS += -i4 -real-size 32

diff --git a/conf/configure.fv3.macosx.gnu b/conf/configure.fv3.macosx.gnu
@@ -1,7 +1,7 @@
 ## NEMS configuration file
 ##
 ## Platform: Darwin Mac OS X
-## Compiler: GNU (clang/gfortran) with MPICH
+## Compiler: GNU (gcc/clang+gfortran) with MPICH
 
 SHELL=/bin/sh
+2 −0		.gitmodules
+1 −1		atmos_model.F90
+1 −1		ccpp/framework
+1 −1		ccpp/physics
+1 −1		gfsphysics/physics/gfdl_cloud_microphys.F90
+0 −0		gfsphysics/physics/module_sf_noahmp_glacier.f90
+0 −0		gfsphysics/physics/module_sf_noahmplsm.f90
+0 −0		gfsphysics/physics/module_wrf_utl.f90
+5 −1		gfsphysics/physics/moninedmf_hafs.f
+0 −0		gfsphysics/physics/noahmp_tables.f90
+14 −14		gfsphysics/physics/samfdeepcnv.f
+7 −4		gfsphysics/physics/satmedmfvdifq.f
+0 −0		gfsphysics/physics/sfc_noahmp_drv.f
+1 −1		io/FV3GFS_io.F90
+1 −1		io/module_write_nemsio.F90
+61 −54		io/module_write_netcdf.F90
+42 −22		io/module_wrt_grid_comp.F90
+11 −3		io/post_gfs.F90