Skip to content

Commit

Permalink
Update to spack-stack-1.5.0 (update to gocart hash f7b34e) (#1920)
Browse files Browse the repository at this point in the history 
* Update modulefiles/ufs_common.lua modulefiles/ufs_hera.gnu.lua modulefiles/ufs_hera.intel.lua for testing ESMF 8.5.0 and MAPL 2.40.3

* Replace [email protected] with [email protected] in ufs_common.lau

* Remove miniconda from tests/rt.sh for spack-stack systems

* Switch to mvapich2 on Hercules with GNU

* Add gfortran-10 argument mismatch compatibility flags in CICE-interface/CMakeLists.txt, needed for mvapich2

* Add export MV2_SHMEM_COLL_NUM_COMM=128 to hercules gnu job card
  • Loading branch information
@climbfuji
climbfuji authored Oct 30, 2023
1 parent 020e783 commit 812aac5
Show file tree
Hide file tree
Showing 25 changed files with 6,808 additions and 6,748 deletions.
3 changes: 3 additions & 0 deletions CICE-interface/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
set(CMAKE_Fortran_FLAGS "-g -fbacktrace")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -ffree-line-length-none ")
if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz")
endif()
set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
set(CMAKE_Fortran_LINK_FLAGS "" )
Expand Down
2 changes: 1 addition & 1 deletion GOCART
Submodule GOCART updated 2 files
+1 −0 CHANGELOG.md
+6 −0 CMakeLists.txt
2 changes: 1 addition & 1 deletion WW3
Submodule WW3 updated 1 files
+8 −1 model/src/w3wavemd.F90
3 changes: 1 addition & 2 deletions modulefiles/ufs_acorn.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@ help([[
Load environment to build UFS on Acorn with Intel compiler
]])

prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

load("stack-intel")
load("stack-cray-mpich")
load("stack-python")

--Avoid production installations; use spack-stack only:
remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/compiler/intel/19.1.3.304")
Expand Down
6 changes: 3 additions & 3 deletions modulefiles/ufs_common.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ local ufs_modules = {
{["crtm"] = "2.4.0"},
{["g2"] = "3.4.5"},
{["g2tmpl"] = "1.10.2"},
{["ip"] = "3.3.3"},
{["ip"] = "4.3.0"},
{["sp"] = "2.3.3"},
{["w3emc"] = "2.9.2"},
{["w3emc"] = "2.10.0"},
{["gftl-shared"] = "1.5.0"},
{["mapl"] = "2.35.2-esmf-8.4.2"},
{["scotch"] = "7.0.3"},
{["scotch"] = "7.0.4"},
}

for i = 1, #ufs_modules do
Expand Down
6 changes: 1 addition & 5 deletions modulefiles/ufs_gaea.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,14 @@ unload("cray-mpich")
unload("cray-python")
unload("darshan")

prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles")
prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c4/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
load(pathJoin("stack-intel", stack_intel_ver))

stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20"
load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
load(pathJoin("stack-python", stack_python_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

Expand Down
6 changes: 2 additions & 4 deletions modulefiles/ufs_hera.gnu.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ help([[
loads UFS Model prerequisites for Hera/GNU
]])

prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
-- for openmpi, need
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")

stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0"
Expand All @@ -11,9 +12,6 @@ load(pathJoin("stack-gcc", stack_gnu_ver))
stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
load(pathJoin("stack-openmpi", stack_openmpi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
load(pathJoin("stack-python", stack_python_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

Expand Down
6 changes: 1 addition & 5 deletions modulefiles/ufs_hera.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,14 @@ help([[
loads UFS Model prerequisites for Hera/Intel
]])

prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
load(pathJoin("stack-python", stack_python_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

Expand Down
39 changes: 5 additions & 34 deletions modulefiles/ufs_hercules.gnu.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,20 @@ help([[
loads UFS Model prerequisites for Hercules/GNU
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230825/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env-mvap2/install/modulefiles/Core")
-- for mvapich2, need:
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles")

stack_gnu_ver=os.getenv("stack_gnu_ver") or "11.3.1"
load(pathJoin("stack-gcc", stack_gnu_ver))

stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
load(pathJoin("stack-openmpi", stack_openmpi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.14"
load(pathJoin("stack-python", stack_python_ver))
stack_mvapich2_ver=os.getenv("stack_mvapich2_ver") or "2.3.7"
load(pathJoin("stack-mvapich2", stack_mvapich2_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

local ufs_modules = {
{["jasper"] = "2.0.32"},
{["zlib"] = "1.2.13"},
{["libpng"] = "1.6.37"},
{["hdf5"] = "1.14.0"},
{["netcdf-c"] = "4.9.2"},
{["netcdf-fortran"] = "4.6.0"},
{["parallelio"] = "2.5.10"},
{["esmf"] = "8.4.2"},
{["fms"] = "2023.01"},
{["bacio"] = "2.4.1"},
{["crtm"] = "2.4.0"},
{["g2"] = "3.4.5"},
{["g2tmpl"] = "1.10.2"},
{["ip"] = "3.3.3"},
{["sp"] = "2.3.3"},
{["w3emc"] = "2.9.2"},
{["gftl-shared"] = "1.5.0"},
{["mapl"] = "2.35.2-esmf-8.4.2"},
{["scotch"] = "7.0.4"},
}

for i = 1, #ufs_modules do
for name, default_version in pairs(ufs_modules[i]) do
local env_version_name = string.gsub(name, "-", "_") .. "_ver"
load(pathJoin(name, os.getenv(env_version_name) or default_version))
end
end
load("ufs_common")

setenv("CC", "mpicc")
setenv("CXX", "mpic++")
Expand Down
35 changes: 2 additions & 33 deletions modulefiles/ufs_hercules.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,18 @@ help([[
loads UFS Model prerequisites for Hercules/Intel
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230825/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles")
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.14"
load(pathJoin("stack-python", stack_python_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

local ufs_modules = {
{["jasper"] = "2.0.32"},
{["zlib"] = "1.2.13"},
{["libpng"] = "1.6.37"},
{["hdf5"] = "1.14.0"},
{["netcdf-c"] = "4.9.2"},
{["netcdf-fortran"] = "4.6.0"},
{["parallelio"] = "2.5.10"},
{["esmf"] = "8.4.2"},
{["fms"] = "2023.01"},
{["bacio"] = "2.4.1"},
{["crtm"] = "2.4.0"},
{["g2"] = "3.4.5"},
{["g2tmpl"] = "1.10.2"},
{["ip"] = "3.3.3"},
{["sp"] = "2.3.3"},
{["w3emc"] = "2.9.2"},
{["gftl-shared"] = "1.5.0"},
{["mapl"] = "2.35.2-esmf-8.4.2"},
{["scotch"] = "7.0.4"},
}

for i = 1, #ufs_modules do
for name, default_version in pairs(ufs_modules[i]) do
local env_version_name = string.gsub(name, "-", "_") .. "_ver"
load(pathJoin(name, os.getenv(env_version_name) or default_version))
end
end
load("ufs_common")

setenv("CC", "mpiicc")
setenv("CXX", "mpiicpc")
Expand Down
6 changes: 1 addition & 5 deletions modulefiles/ufs_jet.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,14 @@ help([[
loads UFS Model prerequisites for Jet/Intel
]])

prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles")
prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
load(pathJoin("stack-python", stack_python_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

Expand Down
37 changes: 37 additions & 0 deletions modulefiles/ufs_linux.intel
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

#%Module######################################################################
##
## UFS prerequisites: Linux (tested: Ubuntu) with gcc/gfortran compilers

echo "Setting environment variables for UFS Model on Linux with gcc/gfortran"

##
## load programming environment: compiler, flags, paths
##
export CC=${CC:-mpicc}
export CXX=${CXX:-mpicxx}
export F77=${F77:-mpif77}
export F90=${F90:-mpif90}
export FC=${FC:-mpif90}

##
## set up variables for ../cmake/configure_linux.intel.cmake
##
export CMAKE_Platform=linux.intel

##
## use own NetCDF library
##
export NETCDF=${NETCDF:-/home/builder/opt}

##
## use pre-compiled EMSF library for above compiler / MPI combination
##
export ESMFMKFILE=${ESMFMKFILE:-/home/builder/opt/lib/esmf.mk}

##
## NCEP libraries (need to download and build manually, see doc/README_{UBUNTU,CENTOS,...}.txt and https://github.com/NCAR/NCEPlibs)
##
export NCEPLIBS_DIR=${NCEPLIBS_DIR:-/home/builder/opt}
export CMAKE_PREFIX_PATH=${NCEPLIBS_DIR}
6 changes: 1 addition & 5 deletions modulefiles/ufs_noaacloud.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ loads UFS Model prerequisites for NOAA Parallelworks/Intel
]])


prepend_path("MODULEPATH", "/contrib/EPIC/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/contrib/spack-stack/modulefiles/core")
prepend_path("MODULEPATH", "/contrib/EPIC/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
load(pathJoin("stack-intel", stack_intel_ver))
Expand All @@ -15,9 +14,6 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
load(pathJoin("stack-python", stack_python_ver))

load("ufs_common")

setenv("CC", "mpiicc")
Expand Down
6 changes: 1 addition & 5 deletions modulefiles/ufs_orion.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,14 @@ help([[
loads UFS Model prerequisites for Orion/Intel
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles")
prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.7"
load(pathJoin("stack-python", stack_python_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

Expand Down
6 changes: 1 addition & 5 deletions modulefiles/ufs_s4.intel.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,14 @@ help([[
loads UFS Model prerequisites for S4/Intel
]])

prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/modulefiles")
prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
load(pathJoin("stack-python", stack_python_ver))

load("ufs_common")

setenv("CC", "mpiicc")
Expand Down
11 changes: 6 additions & 5 deletions tests/fv3_conf/fv3_slurm.IN_hercules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,15 @@ export OMP_NUM_THREADS=@[THRD]
export ESMF_RUNTIME_PROFILE=ON
export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"

# For mvapich2
if [[ @[RT_COMPILER] == gnu ]]; then
export MV2_SHMEM_COLL_NUM_COMM=128
fi

# Avoid job errors because of filesystem synchronization delays
sync && sleep 1

if [[ @[RT_COMPILER] == intel ]]; then
srun --label -n @[TASKS] ./fv3.exe
elif [[ @[RT_COMPILER] == gnu ]]; then
mpirun -n @[TASKS] -N @[TPN] ./fv3.exe
fi
srun --label -n @[TASKS] ./fv3.exe

echo "Model ended: " `date`
echo -n " $( date +%s )," >> job_timestamp.txt
Loading

0 comments on commit 812aac5

Please sign in to comment.