Update merra2_thompson from Dom

.gitignore

@@ -1,4 +1,10 @@
-*.o
+*.[aox]
 *.mod
-*.a
 *.pyc
+
+*.sw[a-p]
+~
+
+build/
+install/
+

.gitmodules

@@ -1,12 +1,14 @@
 [submodule "atmos_cubed_sphere"]
-	path = atmos_cubed_sphere
-	url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere
-	branch = dev/emc
+  path = atmos_cubed_sphere
+  url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere
+  branch = dev/emc
 [submodule "ccpp/framework"]
-	path = ccpp/framework
-	url = https://github.com/NCAR/ccpp-framework
-	branch = main
+  path = ccpp/framework
+  url = https://github.com/NCAR/ccpp-framework
+  branch = main
 [submodule "ccpp/physics"]
-	path = ccpp/physics
-	url = https://github.com/AnningCheng-NOAA/ccpp-physics
-	branch = merra2_thompson
+  path = ccpp/physics
+  #url = https://github.com/NCAR/ccpp-physics
+  #branch = main
+  url = https://github.com/AnningCheng-NOAA/ccpp-physics
+  branch = merra2_thompson

CMakeLists.txt

@@ -6,101 +6,15 @@
 add_subdirectory(ccpp)
 
 ###############################################################################
-### fv3dycore
+### fv3 dynamical core
 ###############################################################################
-list(APPEND _fv3dycore_srcs
-  atmos_cubed_sphere/model/a2b_edge.F90
-  atmos_cubed_sphere/model/multi_gases.F90
-  atmos_cubed_sphere/model/boundary.F90
-  atmos_cubed_sphere/model/dyn_core.F90
-  atmos_cubed_sphere/model/fv_arrays.F90
-  atmos_cubed_sphere/model/fv_control.F90
-  atmos_cubed_sphere/model/fv_dynamics.F90
-  atmos_cubed_sphere/model/fv_fill.F90
-  atmos_cubed_sphere/model/fv_grid_utils.F90
-  atmos_cubed_sphere/model/fv_mapz.F90
-  atmos_cubed_sphere/model/fv_nesting.F90
-  atmos_cubed_sphere/model/fv_regional_bc.F90
-  atmos_cubed_sphere/model/fv_sg.F90
-  atmos_cubed_sphere/model/fv_tracer2d.F90
-  atmos_cubed_sphere/model/fv_update_phys.F90
-  atmos_cubed_sphere/model/sw_core.F90
-  atmos_cubed_sphere/model/tp_core.F90
-  atmos_cubed_sphere/model/nh_core.F90
-  atmos_cubed_sphere/model/nh_utils.F90
-  atmos_cubed_sphere/tools/coarse_grained_diagnostics.F90
-  atmos_cubed_sphere/tools/coarse_grained_restart_files.F90
-  atmos_cubed_sphere/tools/coarse_graining.F90
-  atmos_cubed_sphere/tools/external_ic.F90
-  atmos_cubed_sphere/tools/external_sst.F90
-  atmos_cubed_sphere/tools/fv_diag_column.F90
-  atmos_cubed_sphere/tools/fv_diagnostics.F90
-  atmos_cubed_sphere/tools/fv_eta.F90
-  atmos_cubed_sphere/tools/fv_grid_tools.F90
-  atmos_cubed_sphere/tools/fv_io.F90
-  atmos_cubed_sphere/tools/fv_mp_mod.F90
-  atmos_cubed_sphere/tools/fv_nudge.F90
-  atmos_cubed_sphere/tools/fv_treat_da_inc.F90
-  atmos_cubed_sphere/tools/fv_iau_mod.F90
-  atmos_cubed_sphere/tools/fv_restart.F90
-  atmos_cubed_sphere/tools/fv_surf_map.F90
-  atmos_cubed_sphere/tools/fv_timing.F90
-  atmos_cubed_sphere//tools/init_hydro.F90
-  atmos_cubed_sphere/tools/sim_nc_mod.F90
-  atmos_cubed_sphere/tools/sorted_index.F90
-  atmos_cubed_sphere/tools/test_cases.F90
-  atmos_cubed_sphere/driver/fvGFS/DYCORE_typedefs.F90
-  atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90
-  atmos_cubed_sphere/driver/fvGFS/atmosphere.F90)
-
-add_library(fv3dycore ${_fv3dycore_srcs})
-
-list(APPEND _fv3dycore_defs_private SPMD
-                                    use_WRTCOMP
-                                    GFS_PHYS
-                                    GFS_TYPES
-                                    USE_GFSL63
-                                    MOIST_CAPPA
-                                    USE_COND)
-
-if(MULTI_GASES)
-  list(APPEND _fv3dycore_defs_private MULTI_GASES)
-endif()
-
-if(32BIT)
-  list(APPEND _fv3dycore_defs_private OVERLOAD_R4
-                                      OVERLOAD_R8)
-endif()
-
-if(OPENMP)
-  list(APPEND _fv3dycore_defs_private OPENMP)
-endif()
-
-set_property(SOURCE atmos_cubed_sphere/model/nh_utils.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
-set_property(SOURCE atmos_cubed_sphere/model/fv_mapz.F90  APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
-
-set_target_properties(fv3dycore PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
-target_include_directories(fv3dycore INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
-                                               $<INSTALL_INTERFACE:mod>)
 
-target_compile_definitions(fv3dycore PRIVATE "${_fv3dycore_defs_private}")
-
-#  So much for being consistent:
-# atmos_cubed_sphere/tools/fv_diagnostics.F90: #include <fv_diagnostics.h>
-# atmos_cubed_sphere/tools/fv_eta.F90:         #include <tools/fv_eta.h>
-target_include_directories(fv3dycore PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/atmos_cubed_sphere
-                                             ${CMAKE_CURRENT_SOURCE_DIR}/atmos_cubed_sphere/tools)
-
-target_link_libraries(fv3dycore PUBLIC fms
-                                       fv3ccpp
-                                       esmf)
-if(OPENMP)
-  target_link_libraries(fv3dycore PUBLIC OpenMP::OpenMP_Fortran)
-endif()
-
-# This should not be necessary once framework and physics targets define BUILD_INTERFACE
-target_include_directories(fv3dycore PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/framework/src
-                                             ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics)
+# These ifdefs need to be turned ON in the dycore.
+set(use_WRTCOMP ON)
+set(GFS_PHYS    ON)
+set(GFS_TYPES   ON)
+set(USE_GFSL63  ON)
+add_subdirectory(atmos_cubed_sphere)
 
 ###############################################################################
 ### fv3atm
@@ -134,6 +48,8 @@ add_library(fv3atm
   ${POST_SRC}
 )
 
+add_dependencies(fv3atm fv3 fv3ccpp stochastic_physics)
+
 list(APPEND _fv3atm_defs_private GFS_PHYS
                                  INTERNAL_FILE_NML
                                  use_WRTCOMP)
@@ -144,10 +60,7 @@ set_target_properties(fv3atm PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT
 target_include_directories(fv3atm INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
                                             $<INSTALL_INTERFACE:mod>)
 
-# This should not be necessary once framework and physics targets define BUILD_INTERFACE
-target_include_directories(fv3atm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics)
-
-target_link_libraries(fv3atm PUBLIC fv3dycore
+target_link_libraries(fv3atm PUBLIC fv3
                                     fv3ccpp
                                     stochastic_physics
                                     fms)
@@ -169,7 +82,7 @@ endif()
 ### Install
 ###############################################################################
 install(
-  TARGETS fv3atm fv3dycore fv3ccpp
+  TARGETS fv3atm
   EXPORT fv3atm-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)

atmos_model.F90

@@ -47,12 +47,8 @@ module atmos_model_mod
 use mpp_mod,            only: FATAL, mpp_min, mpp_max, mpp_error, mpp_chksum
 use mpp_domains_mod,    only: domain2d
 use mpp_mod,            only: mpp_get_current_pelist_name
-#ifdef INTERNAL_FILE_NML
 use mpp_mod,            only: input_nml_file
-#else
-use fms_mod,            only: open_namelist_file
-#endif
-use fms_mod,            only: file_exist, error_mesg
+use fms2_io_mod,        only: file_exists
 use fms_mod,            only: close_file, write_version_number, stdlog, stdout
 use fms_mod,            only: clock_flag_default
 use fms_mod,            only: check_nml_error
@@ -551,19 +547,9 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
 !----------------------------------------------------------------------------------------------
 ! initialize atmospheric model - must happen AFTER atmosphere_init so that nests work correctly
 
-   IF ( file_exist('input.nml')) THEN
-#ifdef INTERNAL_FILE_NML
+   IF ( file_exists('input.nml')) THEN
       read(input_nml_file, nml=atmos_model_nml, iostat=io)
       ierr = check_nml_error(io, 'atmos_model_nml')
-#else
-      unit = open_namelist_file ( )
-      ierr=1
-      do while (ierr /= 0)
-         read  (unit, nml=atmos_model_nml, iostat=io, end=10)
-         ierr = check_nml_error(io,'atmos_model_nml')
-      enddo
- 10     call close_file (unit)
-#endif
    endif
 
 !-----------------------------------------------------------------------
@@ -650,6 +636,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
    Init_parm%hydrostatic     = Atm(mygrid)%flagstruct%hydrostatic
 
 #ifdef INTERNAL_FILE_NML
+   ! allocate required to work around GNU compiler bug 100886 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100886
+   allocate(Init_parm%input_nml_file, mold=input_nml_file)
    Init_parm%input_nml_file  => input_nml_file
    Init_parm%fn_nml='using internal file'
 #else
@@ -697,8 +685,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
                               GFS_data%Coupling, GFS_data%Grid, GFS_data%Tbd, GFS_data%Cldprop,  GFS_data%Radtend, &
                               GFS_data%IntDiag, Init_parm, GFS_Diag)
    call FV3GFS_restart_read (GFS_data, GFS_restart_var, Atm_block, GFS_control, Atmos%domain, Atm(mygrid)%flagstruct%warm_start)
-   if(GFS_control%ca_sgs)then
-      call read_ca_restart (Atmos%domain,GFS_control%scells)
+   if(GFS_control%do_ca .and. Atm(mygrid)%flagstruct%warm_start)then
+      call read_ca_restart (Atmos%domain,GFS_control%scells,GFS_control%nca,GFS_control%ncells_g,GFS_control%nca_g)
    endif
    ! Populate the GFS_data%Statein container with the prognostic state
    ! in Atm_block, which contains the initial conditions/restart data.
@@ -925,13 +913,14 @@ subroutine update_atmos_model_state (Atmos, rc)
       call FV3GFS_diag_output(Atmos%Time, GFS_Diag, Atm_block, GFS_control%nx, GFS_control%ny, &
                             GFS_control%levs, 1, 1, 1.0_GFS_kind_phys, time_int, time_intfull, &
                             GFS_control%fhswr, GFS_control%fhlwr)
-      if (nint(GFS_control%fhzero) > 0) then
-        if (mod(isec,3600*nint(GFS_control%fhzero)) == 0) diag_time = Atmos%Time
-      else
-        if (mod(isec,nint(3600*GFS_control%fhzero)) == 0) diag_time = Atmos%Time
-      endif
-      call diag_send_complete_instant (Atmos%Time)
     endif
+    if (nint(GFS_control%fhzero) > 0) then
+      if (mod(isec,3600*nint(GFS_control%fhzero)) == 0) diag_time = Atmos%Time
+    else
+      if (mod(isec,nint(3600*GFS_control%fhzero)) == 0) diag_time = Atmos%Time
+    endif
+    call diag_send_complete_instant (Atmos%Time)
+
 
     !--- this may not be necessary once write_component is fully implemented
     !!!call diag_send_complete_extra (Atmos%Time)
@@ -991,8 +980,8 @@ subroutine atmos_model_end (Atmos)
         GFS_Control%lndp_type > 0  .or. GFS_Control%do_ca ) then
       if(restart_endfcst) then
         call write_stoch_restart_atm('RESTART/atm_stoch.res.nc')
-        if (GFS_control%ca_sgs)then
-          call write_ca_restart(Atmos%domain,GFS_control%scells)
+        if (GFS_control%do_ca)then
+          call write_ca_restart()
         endif
       endif
       call stochastic_physics_wrapper_end(GFS_control)
@@ -1020,8 +1009,8 @@ subroutine atmos_model_restart(Atmos, timestamp)
     call atmosphere_restart(timestamp)
     call FV3GFS_restart_write (GFS_data, GFS_restart_var, Atm_block, &
                                GFS_control, Atmos%domain, timestamp)
-    if(GFS_control%ca_sgs)then
-       call write_ca_restart(Atmos%domain,GFS_control%scells,timestamp)
+    if(GFS_control%do_ca)then
+       call write_ca_restart(timestamp)
     endif
 end subroutine atmos_model_restart
 ! </SUBROUTINE>
@@ -1785,7 +1774,7 @@ subroutine assign_importdata(jdat, rc)
           fldname = 'sea_surface_temperature'
           if (trim(impfield_name) == trim(fldname)) then
             findex  = queryImportFields(fldname)
-            if (importFieldsValid(findex)) then
+            if (importFieldsValid(findex) .and. GFS_control%cplocn2atm) then
 !$omp parallel do default(shared) private(i,j,nb,ix)
               do j=jsc,jec
                 do i=isc,iec
@@ -2337,7 +2326,7 @@ subroutine assign_importdata(jdat, rc)
               do i=isc,iec
                 nb = Atm_block%blkno(i,j)
                 ix = Atm_block%ixp(i,j)
-                GFS_data(nb)%Sfcprop%vtype(ix) = datar82d(i-isc+1,j-jsc+1)
+                GFS_data(nb)%Sfcprop%vtype(ix) = int(datar82d(i-isc+1,j-jsc+1))
               enddo
             enddo
           endif
@@ -2352,7 +2341,7 @@ subroutine assign_importdata(jdat, rc)
               do i=isc,iec
                 nb = Atm_block%blkno(i,j)
                 ix = Atm_block%ixp(i,j)
-                GFS_data(nb)%Sfcprop%stype(ix) = datar82d(i-isc+1,j-jsc+1)
+                GFS_data(nb)%Sfcprop%stype(ix) = int(datar82d(i-isc+1,j-jsc+1))
               enddo
             enddo
           endif

ccpp/CMakeLists.txt

@@ -4,10 +4,6 @@ project(CCPP-FV3
         LANGUAGES C CXX Fortran)
 set(PROJECT "CCPP-FV3")
 
-# Attempt to add link library "NetCDF::NetCDF_Fortran" to target "ccppphys"
-# which is not built in this directory.
-cmake_policy(SET CMP0079 NEW)
-
 #------------------------------------------------------------------------------
 # Set a default build type if none was specified
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
@@ -94,12 +90,7 @@ endif()
 # Build CCPP framework and physics
 
 add_subdirectory(framework)
-
 add_subdirectory(physics)
-add_dependencies(ccppphys ccpp)
-target_link_libraries(ccppphys PUBLIC w3nco::w3nco_d NetCDF::NetCDF_Fortran)
-# This should not be necessary once framework and physics targets define BUILD_INTERFACE
-target_include_directories(ccppphys PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/framework/src)
 
 #------------------------------------------------------------------------------
 # Build fv3ccpp
@@ -124,16 +115,26 @@ add_library(
 set_property(SOURCE driver/GFS_diagnostics.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-O0")
 
 
-target_link_libraries(fv3ccpp PUBLIC ccpp)
-target_link_libraries(fv3ccpp PUBLIC ccppphys)
+target_link_libraries(fv3ccpp PUBLIC ccpp_framework)
+target_link_libraries(fv3ccpp PUBLIC ccpp_physics)
 
 if(OPENMP)
   target_link_libraries(fv3ccpp PUBLIC OpenMP::OpenMP_Fortran)
 endif()
 
-# This should not be necessary once framework and physics targets define BUILD_INTERFACE
-target_include_directories(fv3ccpp PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/framework/src
-                                           ${CMAKE_CURRENT_BINARY_DIR}/physics)
-
 set_target_properties(fv3ccpp PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(fv3ccpp PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
+
+###############################################################################
+### Install
+###############################################################################
+install(
+  TARGETS fv3ccpp ccpp_framework ccpp_physics
+  EXPORT fv3ccpp-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+install(EXPORT fv3ccpp-config
+  DESTINATION lib/cmake)