Merge pull request NOAA-EMC#52 from climbfuji/update_dtc_develop_from…

…_ncar_master_20200606

Update dtc develop from ncar master 2020/06/06

CMakeLists.txt

@@ -1,20 +1,9 @@
-if( NOT DEFINED PHYS)
-  set(PHYS gfs)
-endif()
-
-if (PHYS MATCHES "nam")
-  add_definitions(-DNAM_phys)
-endif()
-
 if(CCPP)
   add_definitions(-DCCPP)
-  if(STATIC)
-    add_definitions(-DSTATIC)
-  endif()
 endif()
 
 add_subdirectory(cpl)
-add_subdirectory(${PHYS}physics)
+add_subdirectory(gfsphysics)
 add_subdirectory(ipd)
 add_subdirectory(io)
 
@@ -73,7 +62,7 @@ set_property(SOURCE atmos_cubed_sphere/model/nh_utils.F90 APPEND_STRING PROPERTY
 set_property(SOURCE atmos_cubed_sphere/model/fv_mapz.F90  APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
 
 target_link_libraries(fv3core PRIVATE fms)
-target_link_libraries(fv3core PRIVATE ${PHYS}physics)
+target_link_libraries(fv3core PRIVATE gfsphysics)
 target_link_libraries(fv3core PRIVATE ipd)
 
 target_include_directories(fv3core PRIVATE ${ESMF_MOD})
@@ -86,7 +75,7 @@ target_include_directories(fv3core PRIVATE ${ESMF_MOD})
 if(CCPP)
   add_subdirectory(ccpp)
   add_subdirectory(ccpp/driver)
-  add_dependencies(${PHYS}physics ccpp ccppphys)
+  add_dependencies(gfsphysics ccpp ccppphys)
   add_dependencies(ccppdriver ccpp ccppphys)
   add_dependencies(ccppphys ccpp)
 
@@ -121,7 +110,7 @@ target_include_directories(fv3cap PRIVATE ${ESMF_MOD})
 
 target_link_libraries(fv3cap PRIVATE fms)
 target_link_libraries(fv3cap PRIVATE fv3cpl)
-target_link_libraries(fv3cap PRIVATE ${PHYS}physics)
+target_link_libraries(fv3cap PRIVATE gfsphysics)
 target_link_libraries(fv3cap PRIVATE ipd)
 target_link_libraries(fv3cap PRIVATE io)
 target_link_libraries(fv3cap PRIVATE fv3core)

atmos_model.F90

@@ -312,14 +312,18 @@ subroutine update_atmos_radiation_physics (Atmos)
     end if
 
     if(IPD_Control%do_ca)then
-       ! DH* The current implementation of cellular_automata assumes that all blocksizes are the
-       ! same, this is tested in the initialization call to cellular_automata, no need to redo *DH
-       call cellular_automata(IPD_Control%kdt, IPD_Data(:)%Statein, IPD_Data(:)%Coupling, IPD_Data(:)%Intdiag, &
-                              Atm_block%nblks, IPD_Control%levs, IPD_Control%nca, IPD_Control%ncells,          &
-                              IPD_Control%nlives, IPD_Control%nfracseed, IPD_Control%nseed,                    &
-                              IPD_Control%nthresh, IPD_Control%ca_global, IPD_Control%ca_sgs,                  &
-                              IPD_Control%iseed_ca, IPD_Control%ca_smooth, IPD_Control%nspinup,                &
-                              Atm_block%blksz(1))
+       if(IPD_Control%ca_sgs)then
+          call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
+            IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,&
+            IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
+            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1))
+       endif
+       if(IPD_Control%ca_global)then
+          call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
+            IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,&
+            IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
+            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude)
+      endif
     endif
 
 !--- if coupled, assign coupled fields
@@ -656,12 +660,21 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
          call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes')
       end if
       ! *DH
-      call cellular_automata(IPD_Control%kdt, IPD_Data(:)%Statein, IPD_Data(:)%Coupling, IPD_Data(:)%Intdiag, &
-                             Atm_block%nblks, IPD_Control%levs, IPD_Control%nca, IPD_Control%ncells,          &
-                             IPD_Control%nlives, IPD_Control%nfracseed, IPD_Control%nseed,                    &
-                             IPD_Control%nthresh, IPD_Control%ca_global, IPD_Control%ca_sgs,                  &
-                             IPD_Control%iseed_ca, IPD_Control%ca_smooth, IPD_Control%nspinup,                &
-                             Atm_block%blksz(1))
+      if(IPD_Control%do_ca)then
+       if(IPD_Control%ca_sgs)then
+          call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
+            IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,&
+            IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
+            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1))
+       endif
+       if(IPD_Control%ca_global)then
+          call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
+            IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,&
+            IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
+            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude)
+       endif
+
+    endif
    endif
 
    Atm(mytile)%flagstruct%do_skeb = IPD_Control%do_skeb
@@ -2044,7 +2057,7 @@ subroutine setup_exportdata (rc)
 
     if (IPD_Control%cplflx) then
     ! MEAN Zonal compt of momentum flux (N/m**2)
-    idx = queryfieldlist(exportFieldsList,'mean_zonal_moment_flx')
+    idx = queryfieldlist(exportFieldsList,'mean_zonal_moment_flx_atm')
     if (idx > 0 ) then
 !$omp parallel do default(shared) private(i,j,nb,ix)
       do j=jsc,jec
@@ -2057,7 +2070,7 @@ subroutine setup_exportdata (rc)
     endif
 
     ! MEAN Merid compt of momentum flux (N/m**2)
-    idx = queryfieldlist(exportFieldsList,'mean_merid_moment_flx')
+    idx = queryfieldlist(exportFieldsList,'mean_merid_moment_flx_atm')
     if (idx > 0 ) then
 !$omp parallel do default(shared) private(i,j,nb,ix)
       do j=jsc,jec

ccpp/CMakeLists.txt

@@ -171,27 +171,13 @@ if (MPI)
   ADD_DEFINITIONS(-DMPI)
 endif (MPI)
 
-#------------------------------------------------------------------------------
-# Set flags for static building/linking of CCPP
-if (STATIC)
-  ADD_DEFINITIONS(-DSTATIC)
-endif (STATIC)
-
 #------------------------------------------------------------------------------
 # Set flag for 32bit dynamics build
 if (DYN32)
   message(STATUS "Dynamics compiled with 32-bit option, adjust fv_sat_adj types")
   ADD_DEFINITIONS(-DOVERLOAD_R4)
 endif (DYN32)
 
-#------------------------------------------------------------------------------
-# If a dynamic build is requested, set TARGET_SUPPORTS_SHARED_LIBS to TRUE; on
-# some systems, cmake doesn't set this even though the system supports shared
-# libraries (e.g. Cray systems)
-if(NOT STATIC)
-SET_PROPERTY(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
-endif(NOT STATIC)
-
 #------------------------------------------------------------------------------
 # Add model-specific flags for C/C++/Fortran preprocessor
 ADD_DEFINITIONS(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM)

ccpp/build_ccpp.sh

@@ -5,8 +5,8 @@ set -eu
 
 # List of valid/tested machines
 VALID_MACHINES=( wcoss_cray wcoss_dell_p3 gaea.intel jet.intel \
-                 hera.intel hera.gnu \
-                 cheyenne.intel cheyenne.intel-impi cheyenne.gnu cheyenne.pgi endeavor.intel \
+                 hera.intel hera.gnu orion.intel \
+                 cheyenne.intel cheyenne.intel-impi cheyenne.gnu endeavor.intel \
                  stampede.intel supermuc_phase2.intel macosx.gnu \
                  linux.intel linux.gnu linux.pgi )
 
@@ -25,11 +25,9 @@ function usage   {
   echo "                                   REPRO=Y/N       (default N)"
   echo "                                   OPENMP=Y/N      (default Y)"
   echo "                                   32BIT=Y/N       (default N, affects dynamics/fast physics only)"
-  echo "                                   STATIC=Y/N      (default N, STATIC=Y requires SUITES=...)"
   echo "                                   SUITES=ABC,XYZ  (comma-separated list of CCPP suites; "
   echo "                                                    corresponding filenames: suite_ABC.xml. ...)"
   echo "                                   MULTI_GASES=Y/N (default N)"
-  echo "                                   INTEL16=Y/N     (default N)"
   echo "           clean_before [optional] can be 'YES' (default) or 'NO'"
   echo "           clean_after  [optional] can be 'YES' (default) or 'NO'"
   exit 1
@@ -125,31 +123,14 @@ if [[ "${MAKE_OPT}" == *"32BIT=Y"* ]]; then
 else
   CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DDYN32=OFF"
 fi
-if [[ "${MAKE_OPT}" == *"STATIC=Y"* ]]; then
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DSTATIC=ON"
-else
-  # hera.gnu uses the NCEPLIBS-external/NCEPLIBS umbrella build libraries,
-  # which cannot be linked dynamically at this point (missing -fPIC flag)
-  if [[ "${MACHINE_ID}" == "hera.gnu" ]]; then
-    echo "Dynamic CCPP build not supported on hera.gnu at this time."
-    exit 1
-  fi
-  # Dynamic builds require linking the NCEPlibs, provide path to them
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DSTATIC=OFF -DBACIO_LIB4=${BACIO_LIB4} -DSP_LIBd=${SP_LIBd} -DW3NCO_LIBd=${W3NCO_LIBd}"
-fi
 if [[ "${MAKE_OPT}" == *"MULTI_GASES=Y"* ]]; then
   CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DMULTI_GASES=ON"
 else
   CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DMULTI_GASES=OFF"
 fi
-if [[ "${MAKE_OPT}" == *"INTEL16=Y"* ]]; then
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DLEGACY_INTEL=ON"
-elif [[ "${MACHINE_ID}" == "wcoss_cray" ]]; then
-  echo "In ccpp_build.sh: flag to cmake that wcoss_cray uses Intel 16"
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DLEGACY_INTEL=ON"
-else
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DLEGACY_INTEL=OFF"
-fi
+
+# Flag to cmake that modern Intel compilers are used
+CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DLEGACY_INTEL=OFF"
 
 # Generate additional CCPP cmake flags depending on machine / compiler
 if [[ "${MACHINE_ID}" == "macosx.gnu" ]]; then
@@ -170,20 +151,6 @@ elif [[ "${MACHINE_ID}" == "gaea.intel" || "${MACHINE_ID}" == "wcoss_cray" ]]; t
   if [[ "${MACHINE_ID}" == "gaea.intel" ]]; then
     CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DLIBXML2_LIB_DIR=${LIBXML2_LIB_DIR} -DLIBXML2_INCLUDE_DIR=${LIBXML2_INCLUDE_DIR}"
   fi
-  # DH* At this time, it is not possible to use the dynamic CCPP
-  # build on gaea/wcoss_cray. While compiling/linking works, the model
-  # crashes immediately. This may be related to 64bit/32bit mismatches
-  # in the MPI libraries (missing "-fPIC" flags when the MPI libraries
-  # were compiled on the system?) - to be investigated.
-  if [[ "${MAKE_OPT}" == *"STATIC=Y"* ]]; then
-    :
-  else
-    ## FOR DYNAMIC BUILD, SET ENVIRONMENT VARIABLE CRAYPE_LINK_TYPE
-    #export CRAYPE_LINK_TYPE=dynamic
-    echo "Dynamic CCPP build not supported on gaea/wcoss_cray at this time."
-    exit 1
-  fi
-  # *DH
 fi
 
 CCPP_CMAKE_FLAGS=$(trim "${CCPP_CMAKE_FLAGS}")
@@ -212,19 +179,10 @@ make ${CCPP_MAKE_FLAGS} install
 # Generate ESMF makefile fragment
 
 set -u
-if ( echo "${MAKE_OPT}" | grep STATIC=Y ) ; then
-  # Set linker flags for static build
-  CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys"
-else
-  # Explicitly append libxml2, with or without path
-  CCPP_XML2_LIB="${LIBXML2_LIB_DIR:+-L${LIBXML2_LIB_DIR} }-lxml2"
-  # Set link objects
-  if ( echo "$MACHINE_ID" | grep gaea ) ; then
-    CCPP_LINK_OBJS="-dynamic -L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB} ${CRAY_PMI_POST_LINK_OPTS} -lpmi"
-  else
-    CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB}"
-  fi
-fi
+
+# Set linker flags
+CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys"
+
 echo "ESMF_DEP_INCPATH=${PATH_CCPP_INC} ${PATH_CCPP_BUILD}/physics" > ${CCPP_MK}
 echo "ESMF_DEP_LINK_OBJS=${CCPP_LINK_OBJS}" >> ${CCPP_MK}