From cbed8f8fba48536dfb5c32b2590e6f749b517a69 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Tue, 22 Oct 2024 13:50:32 -0400 Subject: [PATCH 1/5] Update scripts to build on Gaea-C5 and C6 --- ...ea.intel.lua => gsiutils_gaeac5.intel.lua} | 6 ++--- modulefiles/gsiutils_gaeac6.intel.lua | 24 +++++++++++++++++++ ush/detect_machine.sh | 16 +++++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) rename modulefiles/{gsiutils_gaea.intel.lua => gsiutils_gaeac5.intel.lua} (81%) create mode 100644 modulefiles/gsiutils_gaeac6.intel.lua diff --git a/modulefiles/gsiutils_gaea.intel.lua b/modulefiles/gsiutils_gaeac5.intel.lua similarity index 81% rename from modulefiles/gsiutils_gaea.intel.lua rename to modulefiles/gsiutils_gaeac5.intel.lua index dbc9f8cc..21c807a7 100644 --- a/modulefiles/gsiutils_gaea.intel.lua +++ b/modulefiles/gsiutils_gaeac5.intel.lua @@ -4,8 +4,8 @@ help([[ prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") local python_ver=os.getenv("python_ver") or "3.11.6" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" -local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" @@ -21,4 +21,4 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") -whatis("Description: GSI utilities environment on Gaea with Intel Compilers") +whatis("Description: GSI utilities environment on GaeaC5 with Intel Compilers") diff --git a/modulefiles/gsiutils_gaeac6.intel.lua b/modulefiles/gsiutils_gaeac6.intel.lua new file mode 100644 index 00000000..21c807a7 --- /dev/null +++ b/modulefiles/gsiutils_gaeac6.intel.lua @@ -0,0 +1,24 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") + +local python_ver=os.getenv("python_ver") or "3.11.6" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) +load(pathJoin("python", python_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gsiutils_common") + +load(pathJoin("prod_util", prod_util_ver)) + +pushenv("CFLAGS", "-xHOST") +pushenv("FFLAGS", "-xHOST") + +whatis("Description: GSI utilities environment on GaeaC5 with Intel Compilers") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index b049a604..a2867769 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -21,8 +21,11 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 + + gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 + gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 @@ -81,9 +84,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 ]]; then + # We are on GAEAC5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 ]]; then + # We are on GAEAC6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 From 3308c4475ba81c25bf314afc00510dc2406ab2b7 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Tue, 22 Oct 2024 18:42:08 +0000 Subject: [PATCH 2/5] updates for container --- modulefiles/gsiutils_container.intel.lua | 23 +++++++++++++++++ ush/detect_machine.sh | 10 ++++--- ush/module-setup.sh | 33 ++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 modulefiles/gsiutils_container.intel.lua diff --git a/modulefiles/gsiutils_container.intel.lua b/modulefiles/gsiutils_container.intel.lua new file mode 100644 index 00000000..e95d7115 --- /dev/null +++ b/modulefiles/gsiutils_container.intel.lua @@ -0,0 +1,23 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core") + +--local python_ver=os.getenv("python_ver") or "3.11.6" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.12.2" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gsiutils_common") + +load(pathJoin("prod_util", prod_util_ver)) + +pushenv("CFLAGS", "-march=ivybridge") +pushenv("FFLAGS", "-march=ivybridge") + +whatis("Description: GSI utilities environment in a container with Intel Compilers") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index b049a604..43739793 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -45,7 +45,7 @@ case $(hostname -f) in *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac -if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then +if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then case ${PW_CSP:-} in "aws" | "google" | "azure") MACHINE_ID=noaacloud ;; *) PW_CSP="UNKNOWN" @@ -61,7 +61,9 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then fi # Try searching based on paths since hostname may not match on compute nodes -if [[ -d /lfs/h3 ]]; then +if [[ -d /opt/spack-stack ]]; then + MACHINE_ID=container +elif [[ -d /lfs/h3 ]]; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then @@ -75,8 +77,8 @@ elif [[ -d /scratch1 ]]; then MACHINE_ID=hera elif [[ -d /work ]]; then # We are on MSU Orion or Hercules - mount=$(findmnt -n -o SOURCE /home) - if [[ ${mount} =~ "hercules" ]]; then + if [[ -d /apps/other ]]; then + # We are on Hercules MACHINE_ID=hercules else MACHINE_ID=orion diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 3d738b51..4bb018b1 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -7,6 +7,11 @@ if [[ $MACHINE_ID = jet* ]] ; then source /apps/lmod/lmod/init/bash fi module purge +elif [[ $MACHINE_ID = container* ]] ; then + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /usr/lmod/lmod/init/bash + fi + module purge elif [[ $MACHINE_ID = hera* ]] ; then # We are on NOAA Hera @@ -35,7 +40,6 @@ elif [[ $MACHINE_ID = s4* ]] ; then source /usr/share/lmod/lmod/init/bash fi module purge - elif [[ $MACHINE_ID = wcoss2 ]]; then # We are on WCOSS2 module reset @@ -63,8 +67,33 @@ elif [[ $MACHINE_ID = gaea* ]] ; then # the module command fails. Hence we actually have to source # /etc/profile here. source /etc/profile + __ms_source_etc_profile=yes + else + __ms_source_etc_profile=no + fi + module purge + # clean up after purge + unset _LMFILES_ + unset _LMFILES_000 + unset _LMFILES_001 + unset LOADEDMODULES + module load modules + if [[ -d /opt/cray/ari/modulefiles ]] ; then + module use -a /opt/cray/ari/modulefiles + fi + if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then + module use -a /opt/cray/pe/ari/modulefiles + fi + if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then + module use -a /opt/cray/pe/craype/default/modulefiles + fi + if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then + source /etc/opt/cray/pe/admin-pe/site-config + fi + if [[ "$__ms_source_etc_profile" == yes ]] ; then + source /etc/profile + unset __ms_source_etc_profile fi - module reset elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse From d8c23dda9df5d414188c514287296372dbba01b2 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Tue, 22 Oct 2024 15:06:37 -0400 Subject: [PATCH 3/5] update text in C6 module file --- modulefiles/gsiutils_gaeac6.intel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/gsiutils_gaeac6.intel.lua b/modulefiles/gsiutils_gaeac6.intel.lua index 21c807a7..146c8be8 100644 --- a/modulefiles/gsiutils_gaeac6.intel.lua +++ b/modulefiles/gsiutils_gaeac6.intel.lua @@ -21,4 +21,4 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") -whatis("Description: GSI utilities environment on GaeaC5 with Intel Compilers") +whatis("Description: GSI utilities environment on GaeaC6 with Intel Compilers") From 063c9153ca53c3b5c7fd54e106f0ced1a8a4d56e Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 23 Oct 2024 04:53:34 -0400 Subject: [PATCH 4/5] final adjustments to detect_machine and module-setup --- ush/detect_machine.sh | 6 +++--- ush/module-setup.sh | 28 ++-------------------------- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 033fb6f5..14e7359b 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -48,7 +48,7 @@ case $(hostname -f) in *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac -if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then +if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then case ${PW_CSP:-} in "aws" | "google" | "azure") MACHINE_ID=noaacloud ;; *) PW_CSP="UNKNOWN" @@ -80,8 +80,8 @@ elif [[ -d /scratch1 ]]; then MACHINE_ID=hera elif [[ -d /work ]]; then # We are on MSU Orion or Hercules - if [[ -d /apps/other ]]; then - # We are on Hercules + mount=$(findmnt -n -o SOURCE /home) + if [[ ${mount} =~ "hercules" ]]; then MACHINE_ID=hercules else MACHINE_ID=orion diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 4bb018b1..458b7195 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -40,6 +40,7 @@ elif [[ $MACHINE_ID = s4* ]] ; then source /usr/share/lmod/lmod/init/bash fi module purge + elif [[ $MACHINE_ID = wcoss2 ]]; then # We are on WCOSS2 module reset @@ -67,33 +68,8 @@ elif [[ $MACHINE_ID = gaea* ]] ; then # the module command fails. Hence we actually have to source # /etc/profile here. source /etc/profile - __ms_source_etc_profile=yes - else - __ms_source_etc_profile=no - fi - module purge - # clean up after purge - unset _LMFILES_ - unset _LMFILES_000 - unset _LMFILES_001 - unset LOADEDMODULES - module load modules - if [[ -d /opt/cray/ari/modulefiles ]] ; then - module use -a /opt/cray/ari/modulefiles - fi - if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then - module use -a /opt/cray/pe/ari/modulefiles - fi - if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then - module use -a /opt/cray/pe/craype/default/modulefiles - fi - if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then - source /etc/opt/cray/pe/admin-pe/site-config - fi - if [[ "$__ms_source_etc_profile" == yes ]] ; then - source /etc/profile - unset __ms_source_etc_profile fi + module reset elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse From 2df9611b23015d13bc4cf9f9ac5264c760a4614a Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Mon, 4 Nov 2024 17:45:15 -0500 Subject: [PATCH 5/5] update spack-stack path to C6 install --- modulefiles/gsiutils_gaeac6.intel.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/gsiutils_gaeac6.intel.lua b/modulefiles/gsiutils_gaeac6.intel.lua index 146c8be8..7cf9a72f 100644 --- a/modulefiles/gsiutils_gaeac6.intel.lua +++ b/modulefiles/gsiutils_gaeac6.intel.lua @@ -1,11 +1,11 @@ help([[ ]]) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core") local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" -local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"