ufs-community · zach1221 · Mar 9, 2023 · Feb 3, 2023 · Feb 17, 2023 · Feb 17, 2023
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -63,6 +63,7 @@ set(AVX             OFF CACHE BOOL "Enable AVX-I instruction set")
 set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
 set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
 set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
+set(DISABLE_FMA     OFF CACHE BOOL "Disable Fused Multiply-Add instructions (workaround needed for AMD EPYC)" FORCE)
 set(INLINE_POST     ON  CACHE BOOL "Enable inline post")
 set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
 set(MOVING_NEST     OFF CACHE BOOL "Enable moving nest code")
@@ -137,7 +138,7 @@ if(FMS)
 endif()
 
 if(CMEPS)
-  find_package(PIO 2.5.3 REQUIRED COMPONENTS C Fortran STATIC)
+  find_package(PIO 2.5.3 REQUIRED COMPONENTS C Fortran)
 endif()
 
 find_package(bacio 2.4.0 REQUIRED)

diff --git a/CMakeModules b/CMakeModules
diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake
@@ -43,3 +43,9 @@ if(APPLE)
   # Passing the -c flag includes them and fixes an error with undefined symbols
   set(CMAKE_Fortran_ARCHIVE_FINISH "<CMAKE_RANLIB> -c <TARGET>")
 endif()
+
+# This must be last, to override all other optimization settings.
+if(DISABLE_FMA)
+  set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -no-fma")
+  set(CMAKE_C_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -no-fma")
+endif()
diff --git a/cmake/configure_acorn.cmake b/cmake/configure_acorn.cmake
@@ -1,3 +1,7 @@
 set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
 set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
 set(AVX2            OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
+
+if(FASTER)
+  set(DISABLE_FMA   ON  CACHE BOOL "Disable Fused Multiply-Add instructions (workaround needed for AMD EPYC)" FORCE)
+endif()
diff --git a/cmake/configure_wcoss2.cmake b/cmake/configure_wcoss2.cmake
@@ -1,3 +1,7 @@
 set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
 set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
 set(AVX2            OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
+
+if(FASTER)
+  set(DISABLE_FMA   ON  CACHE BOOL "Disable Fused Multiply-Add instructions (workaround needed for AMD EPYC)" FORCE)
+endif()
diff --git a/modulefiles/ufs_gaea.intel b/modulefiles/ufs_gaea.intel
diff --git a/modulefiles/ufs_gaea.intel.lua b/modulefiles/ufs_gaea.intel.lua
@@ -0,0 +1,35 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea using Intel-2022.1.2
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/role.epic/contrib/modulefiles")
+load(pathJoin("miniconda3",os.getenv("miniconda_ver") or "4.12.0"))
+
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
+
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-2021.3.0_noarch/modulefiles/stack")
+load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+load(pathJoin("intel", os.getenv("intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.11"))
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+
+-- needed for WW3 build
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+-- Needed at runtime:
+load("alps")
+load("rocoto")
+
+load("ufs_common")
+
+setenv("CC","cc")
+setenv("FC","ftn")
+setenv("CXX","CC")
+setenv("CMAKE_C_COMPILER","cc")
+setenv("CMAKE_CXX_COMPILER","CC")
+setenv("CMAKE_Fortran_COMPILER","ftn")
+setenv("CMAKE_Platform","gaea.intel")
diff --git a/modulefiles/ufs_gaea.intel_debug b/modulefiles/ufs_gaea.intel_debug
diff --git a/modulefiles/ufs_gaea.intel_debug.lua b/modulefiles/ufs_gaea.intel_debug.lua
@@ -0,0 +1,35 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea using Intel-2022.1.2
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model and debug on Gaea ]===])
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/role.epic/contrib/modulefiles")
+load(pathJoin("miniconda3",os.getenv("miniconda_ver") or "4.12.0"))
+
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
+
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-2021.3.0_noarch/modulefiles/stack")
+load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+load(pathJoin("intel", os.getenv("intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.11"))
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+
+-- needed for WW3 build
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+-- Needed at runtime:
+load("alps")
+load("rocoto")
+
+load("ufs_common_debug")
+
+setenv("CC","cc")
+setenv("FC","ftn")
+setenv("CXX","CC")
+setenv("CMAKE_C_COMPILER","cc")
+setenv("CMAKE_CXX_COMPILER","CC")
+setenv("CMAKE_Fortran_COMPILER","ftn")
+setenv("CMAKE_Platform","gaea.intel")
diff --git a/tests/compile.sh b/tests/compile.sh
@@ -58,7 +58,7 @@ if [[ $MACHINE_ID == macosx.* ]] || [[ $MACHINE_ID == linux.* ]]; then
 else
   # Activate lua environment for gaea
   if [[ $MACHINE_ID == gaea.* ]] ; then
-    source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh
+    source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
   fi
   # Load fv3 module
   module use $PATHTR/modulefiles
@@ -132,13 +132,13 @@ bash -x ${PATHTR}/build.sh
 
 mv ${BUILD_DIR}/ufs_model ${PATHTR}/tests/${BUILD_NAME}.exe
 if [[ "${MAKE_OPT}" == "-DDEBUG=ON" ]]; then
-  if [[ $MACHINE_ID == gaea.* ]] || [[ $MACHINE_ID == linux.* ]]; then
+  if [[ $MACHINE_ID == linux.* ]]; then
     cp ${PATHTR}/modulefiles/ufs_${MACHINE_ID}_debug ${PATHTR}/tests/modules.${BUILD_NAME}
   else
     cp ${PATHTR}/modulefiles/ufs_${MACHINE_ID}_debug.lua ${PATHTR}/tests/modules.${BUILD_NAME}.lua
   fi
 else
-  if [[ $MACHINE_ID == gaea.* ]] || [[ $MACHINE_ID == linux.* ]]; then
+  if [[ $MACHINE_ID == linux.* ]]; then
     cp ${PATHTR}/modulefiles/ufs_${MACHINE_ID}       ${PATHTR}/tests/modules.${BUILD_NAME}
   else
     cp ${PATHTR}/modulefiles/ufs_${MACHINE_ID}.lua       ${PATHTR}/tests/modules.${BUILD_NAME}.lua

diff --git a/tests/fv3_conf/fv3_qsub.IN_cheyenne b/tests/fv3_conf/fv3_qsub.IN_cheyenne
@@ -17,7 +17,7 @@ if [[ ! -d modulefiles ]]; then
   mkdir modulefiles
   mv *.lua modulefiles
 fi
-module use modulefiles
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2
@@ -15,8 +15,8 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 cd $PBS_O_WORKDIR
 
 set +x
-module use $( pwd -P )
-module load modules.fv3
+module use modulefiles
+module load $PWD/modules.fv3
 module load cray-pals
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_expanse b/tests/fv3_conf/fv3_slurm.IN_expanse
@@ -14,7 +14,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=expanse
 source ./module-setup.sh
-module use $( pwd -P )
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_gaea b/tests/fv3_conf/fv3_slurm.IN_gaea
@@ -15,8 +15,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=gaea
 source ./module-setup.sh
-source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh
-module use $( pwd -P )
+module use --prepend $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_hera b/tests/fv3_conf/fv3_slurm.IN_hera
@@ -16,7 +16,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=hera
 source ./module-setup.sh
-module use $( pwd -P )
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_jet b/tests/fv3_conf/fv3_slurm.IN_jet
@@ -17,7 +17,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=jet
 source ./module-setup.sh
-module use $( pwd -P )
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_orion b/tests/fv3_conf/fv3_slurm.IN_orion
@@ -17,7 +17,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=orion
 source ./module-setup.sh
-module use $( pwd -P )
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_s4 b/tests/fv3_conf/fv3_slurm.IN_s4
@@ -16,7 +16,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=s4
 source ./module-setup.sh
-module use $( pwd -P )
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/fv3_conf/fv3_slurm.IN_stampede b/tests/fv3_conf/fv3_slurm.IN_stampede
@@ -13,7 +13,7 @@ echo -n " $( date +%s )," >  job_timestamp.txt
 set +x
 MACHINE_ID=stampede
 source ./module-setup.sh
-module use $( pwd -P )
+module use $PWD/modulefiles
 module load modules.fv3
 module list
 set -x

diff --git a/tests/module-setup.sh b/tests/module-setup.sh
@@ -60,33 +60,8 @@ elif [[ $MACHINE_ID = gaea* ]] ; then
         # the module command fails.  Hence we actually have to source
         # /etc/profile here.
         source /etc/profile
-        __ms_source_etc_profile=yes
-    else
-        __ms_source_etc_profile=no
-    fi
-    module purge
-    # clean up after purge
-    unset _LMFILES_
-    unset _LMFILES_000
-    unset _LMFILES_001
-    unset LOADEDMODULES
-    module load modules
-    if [[ -d /opt/cray/ari/modulefiles ]] ; then
-        module use -a /opt/cray/ari/modulefiles
-    fi
-    if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then
-        module use -a /opt/cray/pe/ari/modulefiles
-    fi
-    if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then
-        module use -a /opt/cray/pe/craype/default/modulefiles
-    fi
-    if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then
-        source /etc/opt/cray/pe/admin-pe/site-config
-    fi
-    if [[ "$__ms_source_etc_profile" == yes ]] ; then
-        source /etc/profile
-        unset __ms_source_etc_profile
     fi
+    source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
 
 elif [[ $MACHINE_ID = expanse* ]]; then
     # We are on SDSC Expanse

diff --git a/tests/rt.sh b/tests/rt.sh
@@ -446,7 +446,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
 fi
 
 
-BL_DATE=20230227
+BL_DATE=20230308
 
 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 
@@ -821,7 +821,7 @@ else
    echo ; echo REGRESSION TEST WAS SUCCESSFUL
   (echo ; echo REGRESSION TEST WAS SUCCESSFUL) >> ${REGRESSIONTEST_LOG}
 
-  rm -f fv3_*.x fv3_*.exe modules.fv3_* keep_tests.tmp
+  rm -f fv3_*.x fv3_*.exe modules.fv3_* modulefiles/modules.fv3_* keep_tests.tmp
   [[ ${KEEP_RUNDIR} == false ]] && rm -rf ${RUNDIR_ROOT}
   [[ ${ROCOTO} == true ]] && rm -f ${ROCOTO_XML} ${ROCOTO_DB} ${ROCOTO_STATE} *_lock.db
   [[ ${TEST_35D} == true ]] && rm -f tests/cpld_bmark*_20*
+180 −0		Modules/FindGPTL.cmake
+25 −4		Modules/FindNetCDF.cmake
+10 −2		Modules/FindPIO.cmake
+174 −0		Modules/FindPnetCDF.cmake
+32 −0		Modules/FindSphinx.cmake
+47 −0		Modules/Findudunits.cmake