Add support for Gaea C5 (includes PR #1977: new ccpp SDFs added to support RRFS multiphysics ensemble and add tob in ocean output)(Includes PR #1997)

cmake/configure_gaea-c5.intel.cmake

@@ -0,0 +1,4 @@
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(MOM6_Extra_FORTRAN_FLAGS "-xsse2")
+set(HYCOM_Extra_FORTRAN_FLAGS "-xSSE4.2")
+set(HYCOM_Extra_C_FLAGS "-xSSE4.2")

modulefiles/ufs_gaea-c5.intel.lua

@@ -0,0 +1,34 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5 ]===])
+
+load("PrgEnv-intel/8.3.3")
+load("intel-classic/2023.1.0")
+load("cray-mpich/8.1.25")
+load("python/3.9.12")
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.8"
+load(pathJoin("stack-python", stack_python_ver))
+
+load("ufs_common")
+load("nccmp/1.9.0.1")
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaea-c5.intel")

tests/compile.sh

@@ -57,9 +57,12 @@ set +x
 if [[ $MACHINE_ID == macosx ]] || [[ $MACHINE_ID == linux ]]; then
   source $PATHTR/modulefiles/ufs_${MACHINE_ID}.${RT_COMPILER}
 else
-  # Activate lua environment for gaea
+  # Activate lua environment for gaea c4
   if [[ $MACHINE_ID == gaea ]]; then
     source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
+  # Activate lua environment for gaea c5
+  elif [[ $MACHINE_ID == gaea-c5 ]]; then
+    source /lustre/f2/dev/role.epic/contrib/Lmod_init_C5.sh
   fi
   # Load fv3 module
   module use $PATHTR/modulefiles

tests/default_vars.sh

@@ -229,6 +229,19 @@ elif [[ $MACHINE_ID = gaea ]]; then
   INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
   WAV_tasks_atmw_gdas=264
 
+elif [[ $MACHINE_ID = gaea-c5 ]]; then
+
+  TPN=128
+
+  INPES_dflt=3 ; JNPES_dflt=8
+  INPES_thrd=3 ; JNPES_thrd=4
+  INPES_c384=6 ; JNPES_c384=8  ; THRD_c384=1
+  INPES_c768=8 ; JNPES_c768=16 ; THRD_c768=2
+
+  THRD_cpl_atmw_gdas=3
+  INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
+  WAV_tasks_atmw_gdas=264
+
 elif [[ $MACHINE_ID = cheyenne ]]; then
 
   TPN=36

tests/detect_machine.sh

@@ -43,6 +43,15 @@ case $(hostname -f) in
   gaea15.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea15
   gaea16.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea16
 
+  gaea51.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea51
+  gaea52.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea52
+  gaea53.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea53
+  gaea54.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea54
+  gaea55.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea55
+  gaea56.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea56
+  gaea57.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea57
+  gaea58.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea58
+
   hfe01)                   MACHINE_ID=hera ;; ### hera01
   hfe02)                   MACHINE_ID=hera ;; ### hera02
   hfe03)                   MACHINE_ID=hera ;; ### hera03

tests/fv3_conf/compile_slurm.IN_gaea-c5

@@ -0,0 +1,21 @@
+#!/bin/bash -l
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --account=@[ACCNR]
+##SBATCH --qos=@[QUEUE]
+#SBATCH --clusters=es
+#SBATCH --partition=eslogin_c5
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --time=180
+#SBATCH --job-name="@[JBNME]"
+
+set -eux
+
+echo -n " $( date +%s )," >  job_timestamp.txt
+echo "Compile started:  " `date`
+
+@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR] @[RT_COMPILER]
+
+echo "Compile ended:    " `date`
+echo -n " $( date +%s )," >> job_timestamp.txt

tests/fv3_conf/fv3_slurm.IN_gaea-c5

@@ -0,0 +1,38 @@
+#!/bin/bash -l
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --job-name="@[JBNME]"
+#SBATCH --account=@[ACCNR]
+#SBATCH --qos=@[QUEUE]
+#SBATCH --clusters=c5
+#SBATCH --partition=batch
+#SBATCH --nodes=@[NODES]
+#SBATCH --ntasks-per-node=@[TPN]
+#SBATCH --time=@[WLCLK]
+
+set -eux
+echo -n " $( date +%s )," >  job_timestamp.txt
+
+set +x
+MACHINE_ID=gaea-c5
+source ./module-setup.sh
+module use --prepend $PWD/modulefiles
+module load modules.fv3
+module list
+set -x
+
+echo "Model started:  " `date`
+
+export OMP_NUM_THREADS=@[THRD]
+export OMP_STACKSIZE=1024M
+export NC_BLKSZ=1M
+export ESMF_RUNTIME_PROFILE=ON
+export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+
+# Avoid job errors because of filesystem synchronization delays
+sync && sleep 1
+
+srun --label -n @[TASKS] ./fv3.exe
+
+echo "Model ended:    " `date`
+echo -n " $( date +%s )," >> job_timestamp.txt

tests/logs/OpnReqTests_control_p8_hera.log

@@ -1,9 +1,9 @@
-Thu Oct 26 18:02:41 UTC 2023
+Mon Dec  4 14:30:51 UTC 2023
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_bit_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_bit_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_bit_base
 Checking test bit_base control_p8_gnu results ....
 Moving baseline bit_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -51,14 +51,14 @@ Moving baseline bit_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 284.324722
-  0: The maximum resident set size (KB)                   = 1287376
+  0: The total amount of wall time                        = 281.662628
+  0: The maximum resident set size (KB)                   = 1302480
 
 Test bit_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_dbg_base
 Checking test dbg_base control_p8_gnu results ....
 Moving baseline dbg_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -106,14 +106,14 @@ Moving baseline dbg_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 950.779343
-  0: The maximum resident set size (KB)                   = 1271732
+  0: The total amount of wall time                        = 909.259942
+  0: The maximum resident set size (KB)                   = 1289656
 
 Test dbg_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_dcp
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_dcp
 Checking test dcp control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -160,14 +160,14 @@ Checking test dcp control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 257.372254
-  0: The maximum resident set size (KB)                   = 1270624
+  0: The total amount of wall time                        = 249.181311
+  0: The maximum resident set size (KB)                   = 1281316
 
 Test dcp control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_mpi
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_mpi
 Checking test mpi control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -214,14 +214,14 @@ Checking test mpi control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 258.586894
-  0: The maximum resident set size (KB)                   = 1271300
+  0: The total amount of wall time                        = 250.654118
+  0: The maximum resident set size (KB)                   = 1279188
 
 Test mpi control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_rst
 Checking test rst control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -268,14 +268,14 @@ Checking test rst control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 279.166194
-  0: The maximum resident set size (KB)                   = 1271756
+  0: The total amount of wall time                        = 252.596116
+  0: The maximum resident set size (KB)                   = 1279660
 
 Test rst control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_std_base
 Checking test std_base control_p8_gnu results ....
 Moving baseline std_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -323,14 +323,14 @@ Moving baseline std_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 263.273140
-  0: The maximum resident set size (KB)                   = 1271588
+  0: The total amount of wall time                        = 250.001003
+  0: The maximum resident set size (KB)                   = 1280812
 
 Test std_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_110447/control_p8_gnu_thr
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_45674/control_p8_gnu_thr
 Checking test thr control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -377,11 +377,11 @@ Checking test thr control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 269.636095
-  0: The maximum resident set size (KB)                   = 1271256
+  0: The total amount of wall time                        = 250.574652
+  0: The maximum resident set size (KB)                   = 1283668
 
 Test thr control_p8_gnu PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Thu Oct 26 19:33:06 UTC 2023
-Elapsed time: 01h:30m:25s. Have a nice day!
+Mon Dec  4 15:39:13 UTC 2023
+Elapsed time: 01h:08m:22s. Have a nice day!

tests/logs/OpnReqTests_cpld_control_nowave_noaero_p8_hera.log

@@ -1,9 +1,9 @@
-Thu Oct 26 21:28:55 UTC 2023
+Mon Dec  4 13:38:49 UTC 2023
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/cpld_control_c96_noaero_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_65977/cpld_control_nowave_noaero_p8_gnu_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_221447/cpld_control_nowave_noaero_p8_gnu_dbg_base
 Checking test dbg_base cpld_control_nowave_noaero_p8_gnu results ....
 Moving baseline dbg_base cpld_control_nowave_noaero_p8_gnu files ....
  Moving sfcf021.tile1.nc .........OK
@@ -66,14 +66,14 @@ Moving baseline dbg_base cpld_control_nowave_noaero_p8_gnu files ....
  Moving RESTART/iced.2021-03-23-21600.nc .........OK
  Moving RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK
 
-  0: The total amount of wall time                        = 1290.914205
-  0: The maximum resident set size (KB)                   = 1405340
+  0: The total amount of wall time                        = 1255.630972
+  0: The maximum resident set size (KB)                   = 1411232
 
 Test dbg_base cpld_control_nowave_noaero_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/cpld_control_c96_noaero_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_65977/cpld_control_nowave_noaero_p8_gnu_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_221447/cpld_control_nowave_noaero_p8_gnu_rst
 Checking test rst cpld_control_nowave_noaero_p8_gnu results ....
  Comparing sfcf021.tile1.nc .........OK
  Comparing sfcf021.tile2.nc .........OK
@@ -135,14 +135,14 @@ Checking test rst cpld_control_nowave_noaero_p8_gnu results ....
  Comparing RESTART/iced.2021-03-23-21600.nc .........OK
  Comparing RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK
 
-  0: The total amount of wall time                        = 404.189648
-  0: The maximum resident set size (KB)                   = 1396372
+  0: The total amount of wall time                        = 388.631231
+  0: The maximum resident set size (KB)                   = 1406476
 
 Test rst cpld_control_nowave_noaero_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/cpld_control_c96_noaero_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_65977/cpld_control_nowave_noaero_p8_gnu_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_221447/cpld_control_nowave_noaero_p8_gnu_std_base
 Checking test std_base cpld_control_nowave_noaero_p8_gnu results ....
 Moving baseline std_base cpld_control_nowave_noaero_p8_gnu files ....
  Moving sfcf021.tile1.nc .........OK
@@ -205,11 +205,11 @@ Moving baseline std_base cpld_control_nowave_noaero_p8_gnu files ....
  Moving RESTART/iced.2021-03-23-21600.nc .........OK
  Moving RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK
 
-  0: The total amount of wall time                        = 419.610070
-  0: The maximum resident set size (KB)                   = 1397268
+  0: The total amount of wall time                        = 387.397993
+  0: The maximum resident set size (KB)                   = 1400340
 
 Test std_base cpld_control_nowave_noaero_p8_gnu PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Thu Oct 26 22:19:38 UTC 2023
-Elapsed time: 00h:50m:44s. Have a nice day!
+Mon Dec  4 14:28:11 UTC 2023
+Elapsed time: 00h:49m:22s. Have a nice day!