From 7fa8d3e141fd04be4473b35c8e84465197a21b78 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 9 Apr 2022 14:53:25 +0000 Subject: [PATCH 01/26] Pass machine name to build scripts. --- devbuild.sh | 14 ++++-- env/detect_machine.sh | 107 ------------------------------------------ env/set_machine.sh | 6 +++ test/README.md | 2 +- test/build.sh | 10 ++-- 5 files changed, 23 insertions(+), 116 deletions(-) delete mode 100755 env/detect_machine.sh create mode 100755 env/set_machine.sh diff --git a/devbuild.sh b/devbuild.sh index b46063d30f..9581075934 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -3,7 +3,7 @@ # usage instructions usage () { cat << EOF_USAGE -Usage: $0 [OPTIONS]... +Usage: $0 --platform=PLATFORM [OPTIONS]... OPTIONS -h, --help @@ -93,8 +93,6 @@ BUILD_JOBS=4 CLEAN=false CONTINUE=false VERBOSE=false -# detect PLATFORM (MACHINE) -source ${SRC_DIR}/env/detect_machine.sh # process required arguments if [[ ("$1" == "--help") || ("$1" == "-h") ]]; then @@ -138,6 +136,16 @@ while :; do shift done +# check if PLATFORM is set +if [ -z $PLATFORM ] ; then + printf "\nERROR: Please set PLATFORM.\n\n" + usage + exit 0 +fi + +# set PLATFORM (MACHINE) +source ${SRC_DIR}/env/set_machine.sh $PLATFORM + set -eu # automatically determine compiler diff --git a/env/detect_machine.sh b/env/detect_machine.sh deleted file mode 100755 index 1634b18dd1..0000000000 --- a/env/detect_machine.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -case $(hostname -f) in - - v71a1.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### venus - v71a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### venus - v71a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### venus - v72a1.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### venus - v72a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### venus - v72a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### venus - - m71a1.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars - m71a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars - m71a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars - m72a1.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars - m72a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars - m72a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars - - alogin01.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - alogin02.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - adecflow01.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - adecflow02.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - - gaea9) MACHINE_ID=gaea ;; ### gaea9 - gaea10) MACHINE_ID=gaea ;; ### gaea10 - gaea11) MACHINE_ID=gaea ;; ### gaea11 - gaea12) MACHINE_ID=gaea ;; ### gaea12 - gaea13) MACHINE_ID=gaea ;; ### gaea13 - gaea14) MACHINE_ID=gaea ;; ### gaea14 - gaea15) MACHINE_ID=gaea ;; ### gaea15 - gaea16) MACHINE_ID=gaea ;; ### gaea16 - gaea9.ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 - gaea10.ncrc.gov) MACHINE_ID=gaea ;; ### gaea10 - gaea11.ncrc.gov) MACHINE_ID=gaea ;; ### gaea11 - gaea12.ncrc.gov) MACHINE_ID=gaea ;; ### gaea12 - gaea13.ncrc.gov) MACHINE_ID=gaea ;; ### gaea13 - gaea14.ncrc.gov) MACHINE_ID=gaea ;; ### gaea14 - gaea15.ncrc.gov) MACHINE_ID=gaea ;; ### gaea15 - gaea16.ncrc.gov) MACHINE_ID=gaea ;; ### gaea16 - - hfe01) MACHINE_ID=hera ;; ### hera01 - hfe02) MACHINE_ID=hera ;; ### hera02 - hfe03) MACHINE_ID=hera ;; ### hera03 - hfe04) MACHINE_ID=hera ;; ### hera04 - hfe05) MACHINE_ID=hera ;; ### hera05 - hfe06) MACHINE_ID=hera ;; ### hera06 - hfe07) MACHINE_ID=hera ;; ### hera07 - hfe08) MACHINE_ID=hera ;; ### hera08 - hfe09) MACHINE_ID=hera ;; ### hera09 - hfe10) MACHINE_ID=hera ;; ### hera10 - hfe11) MACHINE_ID=hera ;; ### hera11 - hfe12) MACHINE_ID=hera ;; ### hera12 - hecflow01) MACHINE_ID=hera ;; ### heraecflow01 - - s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4 - - fe1) MACHINE_ID=jet ;; ### jet01 - fe2) MACHINE_ID=jet ;; ### jet02 - fe3) MACHINE_ID=jet ;; ### jet03 - fe4) MACHINE_ID=jet ;; ### jet04 - fe5) MACHINE_ID=jet ;; ### jet05 - fe6) MACHINE_ID=jet ;; ### jet06 - fe7) MACHINE_ID=jet ;; ### jet07 - fe8) MACHINE_ID=jet ;; ### jet08 - tfe1) MACHINE_ID=jet ;; ### jet09 - tfe2) MACHINE_ID=jet ;; ### jet10 - - Orion-login-1.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1 - Orion-login-2.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion2 - Orion-login-3.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion3 - Orion-login-4.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion4 - - cheyenne1.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - cheyenne2.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2 - cheyenne3.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3 - cheyenne4.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4 - cheyenne5.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5 - cheyenne6.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6 - cheyenne1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - cheyenne2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2 - cheyenne3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3 - cheyenne4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4 - cheyenne5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5 - cheyenne6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6 - chadmin1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - chadmin2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - chadmin3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - chadmin4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - chadmin5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - chadmin6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - - login1.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1 - login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2 - login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3 - login4.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede4 - - login01.expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1 - login02.expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse2 - - nid00193) MACHINE_ID=odin ;; ### Odin1 at NSSL - nid00385) MACHINE_ID=odin ;; ### Odin2 at NSSL -esac - -MACHINE="${MACHINE_ID}" -PLATFORM="${MACHINE}" - -printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 diff --git a/env/set_machine.sh b/env/set_machine.sh new file mode 100755 index 0000000000..05985d228d --- /dev/null +++ b/env/set_machine.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +MACHINE="$1" +PLATFORM="${MACHINE}" + +printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 diff --git a/test/README.md b/test/README.md index 84f58027eb..96898ba85c 100644 --- a/test/README.md +++ b/test/README.md @@ -33,7 +33,7 @@ On cheyenne: ``` cd test -./build.sh >& build.out & +./build.sh cheyenne >& build.out & ``` Check the ``${SR_WX_APP_TOP_DIR}/test/build_test$PID.out`` file for PASS/FAIL. diff --git a/test/build.sh b/test/build.sh index 186c6b90e0..ea3e8a4ae4 100755 --- a/test/build.sh +++ b/test/build.sh @@ -7,7 +7,7 @@ # # Usage: see function usage below # -# Examples: ./build.sh >& test.out & +# Examples: ./build.sh hera >& test.out & # set -eux # Uncomment for debugging #======================================================================= @@ -16,14 +16,14 @@ fail() { echo -e "\n$1\n" >> ${TEST_OUTPUT} && exit 1; } function usage() { echo - echo "Usage: $0 " + echo "Usage: $0 machine" echo exit 1 } machines=( hera jet cheyenne orion wcoss_cray wcoss_dell_p3 ) -[[ $# -eq 1 ]] && usage +[[ $# -eq 2 ]] && usage #----------------------------------------------------------------------- @@ -35,7 +35,7 @@ TOP_DIR=${TEST_DIR}/.. # Top level (umbrella repo) directory TEST_OUTPUT=${TEST_DIR}/build_test${PID}.out # Detect MACHINE -source ${TOP_DIR}/env/detect_machine.sh +source ${TOP_DIR}/env/set_machine.sh $1 machine=$(echo "${MACHINE}" | tr '[A-Z]' '[a-z]') # scripts in sorc need lower case machine name @@ -100,7 +100,7 @@ declare -a executables_created=( chgres_cube \ BIN_DIR=${TOP_DIR}/bin_${compiler} EXEC_DIR=${BIN_DIR}/bin if [ $build_it -eq 0 ] ; then - ./devbuild.sh --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${BIN_DIR} \ + ./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${BIN_DIR} \ --clean || fail "Build ${machine} ${compiler} FAILED" fi # End of skip build for testing From dbc9d2f07ebe08274e3dd2f38935c4088d7f58a1 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 9 Apr 2022 16:35:13 +0000 Subject: [PATCH 02/26] Use modules environment instead of shell scripts. --- devbuild.sh | 4 +- ...ld_cheyenne_gnu.env => build_cheyenne_gnu} | 17 +++-- ...heyenne_intel.env => build_cheyenne_intel} | 17 +++-- ...{build_hera_intel.env => build_hera_intel} | 19 ++++-- env/build_jet_intel | 29 +++++++++ env/build_jet_intel.env | 22 ------- ...ccgfortran.env => build_macos_gccgfortran} | 0 env/build_odin_intel | 65 +++++++++++++++++++ env/build_odin_intel.env | 58 ----------------- env/build_orion_intel | 29 +++++++++ env/build_orion_intel.env | 22 ------- env/build_singularity_gnu | 45 +++++++++++++ env/build_singularity_gnu.env | 40 ------------ env/build_wcoss_dell_p3_intel | 31 +++++++++ env/build_wcoss_dell_p3_intel.env | 24 ------- env/wflow_cheyenne | 16 +++++ env/wflow_cheyenne.env | 6 -- env/wflow_hera | 21 ++++++ env/wflow_hera.env | 7 -- env/wflow_jet | 21 ++++++ env/wflow_jet.env | 7 -- env/wflow_odin | 47 ++++++++++++++ env/wflow_odin.env | 25 ------- env/wflow_orion | 21 ++++++ env/wflow_orion.env | 7 -- env/wflow_singularity | 16 +++++ env/wflow_singularity.env | 3 - ..._wcoss_dell_p3.env => wflow_wcoss_dell_p3} | 9 ++- 28 files changed, 388 insertions(+), 240 deletions(-) rename env/{build_cheyenne_gnu.env => build_cheyenne_gnu} (51%) rename env/{build_cheyenne_intel.env => build_cheyenne_intel} (51%) rename env/{build_hera_intel.env => build_hera_intel} (57%) create mode 100644 env/build_jet_intel delete mode 100644 env/build_jet_intel.env rename env/{build_macos_gccgfortran.env => build_macos_gccgfortran} (100%) create mode 100644 env/build_odin_intel delete mode 100644 env/build_odin_intel.env create mode 100644 env/build_orion_intel delete mode 100644 env/build_orion_intel.env create mode 100644 env/build_singularity_gnu delete mode 100644 env/build_singularity_gnu.env create mode 100644 env/build_wcoss_dell_p3_intel delete mode 100644 env/build_wcoss_dell_p3_intel.env create mode 100644 env/wflow_cheyenne delete mode 100644 env/wflow_cheyenne.env create mode 100644 env/wflow_hera delete mode 100644 env/wflow_hera.env create mode 100644 env/wflow_jet delete mode 100644 env/wflow_jet.env create mode 100644 env/wflow_odin delete mode 100644 env/wflow_odin.env create mode 100644 env/wflow_orion delete mode 100644 env/wflow_orion.env create mode 100644 env/wflow_singularity delete mode 100644 env/wflow_singularity.env rename env/{wflow_wcoss_dell_p3.env => wflow_wcoss_dell_p3} (51%) diff --git a/devbuild.sh b/devbuild.sh index 9581075934..40067f9319 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -168,7 +168,7 @@ if [ "${VERBOSE}" = true ] ; then fi # set ENV_FILE for this platform/compiler combination -ENV_FILE="${SRC_DIR}/env/build_${PLATFORM}_${COMPILER}.env" +ENV_FILE="${SRC_DIR}/env/build_${PLATFORM}_${COMPILER}" if [ ! -f "${ENV_FILE}" ]; then printf "ERROR: environment file does not exist for platform/compiler\n" >&2 printf " ENV_FILE=${ENV_FILE}\n" >&2 @@ -239,7 +239,7 @@ fi # source the environment file for this platform/compiler combination, then build the code printf "... Source ENV_FILE and create BUILD directory ...\n" module use ${SRC_DIR}/env -. ${ENV_FILE} +module load ${ENV_FILE} module list mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} diff --git a/env/build_cheyenne_gnu.env b/env/build_cheyenne_gnu similarity index 51% rename from env/build_cheyenne_gnu.env rename to env/build_cheyenne_gnu index 6cc15ce757..f6a1111579 100644 --- a/env/build_cheyenne_gnu.env +++ b/env/build_cheyenne_gnu @@ -1,4 +1,11 @@ -#Setup instructions for CISL Cheyenne using GNU-10.1.0 (bash shell) +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the CISL Cheyenne machine using GNU" +} + +module-whatis "Loads libraries needed for building SRW on Cheyenne" module purge @@ -18,8 +25,8 @@ module load hpc-mpt/2.22 module load srw_common module load g2/3.4.3 -export CMAKE_C_COMPILER=mpicc -export CMAKE_CXX_COMPILER=mpicxx -export CMAKE_Fortran_COMPILER=mpif90 -export CMAKE_Platform=cheyenne.gnu +setenv CMAKE_C_COMPILER mpicc +setenv CMAKE_CXX_COMPILER mpicxx +setenv CMAKE_Fortran_COMPILER mpif90 +setenv CMAKE_Platform cheyenne.gnu diff --git a/env/build_cheyenne_intel.env b/env/build_cheyenne_intel similarity index 51% rename from env/build_cheyenne_intel.env rename to env/build_cheyenne_intel index 31dbc3da2b..bfc2c16e23 100644 --- a/env/build_cheyenne_intel.env +++ b/env/build_cheyenne_intel @@ -1,4 +1,11 @@ -#Setup instructions for CISL Cheyenne using Intel-2021.2 (bash shell) +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the CISL Cheyenne machine using Intel-19.1.1" +} + +module-whatis "Loads libraries needed for building SRW on Cheyenne" module purge @@ -18,8 +25,8 @@ module load hpc-mpt/2.22 module load srw_common module load g2/3.4.3 -export CMAKE_C_COMPILER=mpicc -export CMAKE_CXX_COMPILER=mpicxx -export CMAKE_Fortran_COMPILER=mpif90 -export CMAKE_Platform=cheyenne.intel +setenv CMAKE_C_COMPILER mpicc +setenv CMAKE_CXX_COMPILER mpicxx +setenv CMAKE_Fortran_COMPILER mpif90 +setenv CMAKE_Platform cheyenne.intel diff --git a/env/build_hera_intel.env b/env/build_hera_intel similarity index 57% rename from env/build_hera_intel.env rename to env/build_hera_intel index 5eea2338a8..76c317c3a0 100644 --- a/env/build_hera_intel.env +++ b/env/build_hera_intel @@ -1,4 +1,11 @@ -#Setup instructions for NOAA RDHPC Hera using Intel-18.0.5.274 (bash shell) +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA RDHPC machine Hera using Intel-18.0.5.274" +} + +module-whatis "Loads libraries needed for building SRW on Hera" module purge @@ -21,10 +28,10 @@ module load srw_common module unload fms module load fms/2021.03-avx -export CMAKE_C_COMPILER=mpiicc -export CMAKE_CXX_COMPILER=mpiicpc -export CMAKE_Fortran_COMPILER=mpiifort -export CMAKE_Platform=hera.intel +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform hera.intel # use shared memory and OpenFabrics Alliance (OFA) fabric with Intel MPI to circumvent RDMA-related bug in DAPL. -export I_MPI_Platform=shm:ofa +setenv I_MPI_Platform shm:ofa diff --git a/env/build_jet_intel b/env/build_jet_intel new file mode 100644 index 0000000000..405999038a --- /dev/null +++ b/env/build_jet_intel @@ -0,0 +1,29 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA RDHPC machine JET using Intel-18.0.5.274" +} + +module-whatis "Loads libraries needed for building SRW on JET" + +module purge + +module use /contrib/sutils/modulefiles +module load sutils + +module load cmake/3.20.1 + +module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack + +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.4.274 + +module load srw_common + +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform jet.intel + diff --git a/env/build_jet_intel.env b/env/build_jet_intel.env deleted file mode 100644 index 777930d660..0000000000 --- a/env/build_jet_intel.env +++ /dev/null @@ -1,22 +0,0 @@ -#Setup instructions for NOAA RDHPC Jet using Intel-18.0.5.274 (bash shell) - -module purge - -module use /contrib/sutils/modulefiles -module load sutils - -module load cmake/3.20.1 - -module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack - -module load hpc/1.1.0 -module load hpc-intel/18.0.5.274 -module load hpc-impi/2018.4.274 - -module load srw_common - -export CMAKE_C_COMPILER=mpiicc -export CMAKE_CXX_COMPILER=mpiicpc -export CMAKE_Fortran_COMPILER=mpiifort -export CMAKE_Platform=jet.intel - diff --git a/env/build_macos_gccgfortran.env b/env/build_macos_gccgfortran similarity index 100% rename from env/build_macos_gccgfortran.env rename to env/build_macos_gccgfortran diff --git a/env/build_odin_intel b/env/build_odin_intel new file mode 100644 index 0000000000..164b0b32d3 --- /dev/null +++ b/env/build_odin_intel @@ -0,0 +1,65 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA RDHPC machine odin using Intel" +} + +module-whatis "Loads libraries needed for building SRW on odin" + +module unload modules +unset -f module + +setenv BASH_ENV "/usr/local/lmod/8.3.1/init/bash" +puts "source $BASH_ENV;" +setenv LMOD_SYSTEM_DEFAULT_MODULES "PrgEnv-intel:cray-mpich:intel:craype" +module --initial_load --no_redirect restore +#module use <$HOME>/ +setenv MODULEPATH "/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" + +#module purge +setenv CMAKE "/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake" +setenv PATH "/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH}" + +module load hpc/1.2.0 +module load hpc-intel +module load hpc-cray-mpich + +#module load srw_common + +module load jasper +module load zlib +module load png + +#module load cray-hdf5 +#module load cray-netcdf +module load esmf +module load fms + +module load bacio +module load crtm +module load g2 +module load g2tmpl +module load ip +module load sp +module load w3nco +module load upp + +module load gftl-shared +module load yafyaml +module load mapl + +module load gfsio +module load landsfcutil +module load nemsio +module load nemsiogfs +module load sfcio +module load sigio +module load w3emc +module load wgrib2 + +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform odin.intel + diff --git a/env/build_odin_intel.env b/env/build_odin_intel.env deleted file mode 100644 index cf69d3d091..0000000000 --- a/env/build_odin_intel.env +++ /dev/null @@ -1,58 +0,0 @@ -#Setup instructions for NOAA RDHPC Jet using Intel-18.0.5.274 (bash shell) - -module unload modules -unset -f module - -export BASH_ENV=/usr/local/lmod/8.3.1/init/bash -source $BASH_ENV -export LMOD_SYSTEM_DEFAULT_MODULES=PrgEnv-intel:cray-mpich:intel:craype -module --initial_load --no_redirect restore -#module use <$HOME>/ -export MODULEPATH=/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles - -#module purge -export CMAKE=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake -export PATH=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH} - -module load hpc/1.2.0 -module load hpc-intel -module load hpc-cray-mpich - -#module load srw_common - -module load jasper -module load zlib -module load png - -#module load cray-hdf5 -#module load cray-netcdf -module load esmf -module load fms - -module load bacio -module load crtm -module load g2 -module load g2tmpl -module load ip -module load sp -module load w3nco -module load upp - -module load gftl-shared -module load yafyaml -module load mapl - -module load gfsio -module load landsfcutil -module load nemsio -module load nemsiogfs -module load sfcio -module load sigio -module load w3emc -module load wgrib2 - -export CMAKE_C_COMPILER=cc -export CMAKE_CXX_COMPILER=CC -export CMAKE_Fortran_COMPILER=ftn -export CMAKE_Platform=odin.intel - diff --git a/env/build_orion_intel b/env/build_orion_intel new file mode 100644 index 0000000000..7dc4888883 --- /dev/null +++ b/env/build_orion_intel @@ -0,0 +1,29 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the MSU machineOrion using Intel-18.0.5" +} + +module-whatis "Loads libraries needed for building SRW on Orion" + +module purge + +module load contrib noaatools + +module load cmake/3.22.1 +module load python/3.9.2 + +module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack + +module load hpc/1.1.0 +module load hpc-intel/2018.4 +module load hpc-impi/2018.4 + +module load srw_common + +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform orion.intel + diff --git a/env/build_orion_intel.env b/env/build_orion_intel.env deleted file mode 100644 index 24db81de42..0000000000 --- a/env/build_orion_intel.env +++ /dev/null @@ -1,22 +0,0 @@ -#Setup instructions for MSU Orion using Intel-18.0.5 (bash shell) - -module purge - -module load contrib noaatools - -module load cmake/3.22.1 -module load python/3.9.2 - -module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack - -module load hpc/1.1.0 -module load hpc-intel/2018.4 -module load hpc-impi/2018.4 - -module load srw_common - -export CMAKE_C_COMPILER=mpiicc -export CMAKE_CXX_COMPILER=mpiicpc -export CMAKE_Fortran_COMPILER=mpiifort -export CMAKE_Platform=orion.intel - diff --git a/env/build_singularity_gnu b/env/build_singularity_gnu new file mode 100644 index 0000000000..ee58e550be --- /dev/null +++ b/env/build_singularity_gnu @@ -0,0 +1,45 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW in a singularity container" + puts stderr "using Gnu 9.3.0 built from this docker image:" + puts stderr " docker://noaaepic/ubuntu20.04-epic-srwapp:1.0" + puts stderr "https://hub.docker.com/r/noaaepic/ubuntu20.04-epic-srwapp" +} + +module-whatis "Loads libraries needed for building SRW in a singularity container" + +module purge + +module use /opt/hpc-modules/modulefiles/stack + +module load hpc +module load hpc-gnu +module load hpc-openmpi + +module load netcdf +module load hdf5 +module load bacio +module load sfcio +module load sigio +module load nemsio +module load w3emc +module load esmf +module load fms +module load crtm +module load g2 +module load png +module load zlib +module load g2tmpl +module load ip +module load sp +module load w3nco +module load cmake/3.21.1 +module load module load gfsio +module load wgrib2 +module load upp + +setenv CMAKE_C_COMPILER mpicc +setenv CMAKE_CXX_COMPILER mpicxx +setenv CMAKE_Fortran_COMPILER mpif90 +setenv CMAKE_Platform singularity.gnu diff --git a/env/build_singularity_gnu.env b/env/build_singularity_gnu.env deleted file mode 100644 index dc1c65265e..0000000000 --- a/env/build_singularity_gnu.env +++ /dev/null @@ -1,40 +0,0 @@ -#Setup instructions for singularity container using gnu 9.3.0 built from this docker image: docker://noaaepic/ubuntu20.04-epic-srwapp:1.0 -#https://hub.docker.com/r/noaaepic/ubuntu20.04-epic-srwapp - -source /usr/share/lmod/6.6/init/profile - -module purge - -module use /opt/hpc-modules/modulefiles/stack - -module load hpc -module load hpc-gnu -module load hpc-openmpi - -module load netcdf -module load hdf5 -module load bacio -module load sfcio -module load sigio -module load nemsio -module load w3emc -module load esmf -module load fms -module load crtm -module load g2 -module load png -module load zlib -module load g2tmpl -module load ip -module load sp -module load w3nco -module load cmake/3.21.1 -module load module load gfsio -module load wgrib2 -module load upp - - -export CMAKE_C_COMPILER=mpicc -export CMAKE_CXX_COMPILER=mpicxx -export CMAKE_Fortran_COMPILER=mpif90 -export CMAKE_Platform=singularity.gnu diff --git a/env/build_wcoss_dell_p3_intel b/env/build_wcoss_dell_p3_intel new file mode 100644 index 0000000000..7668396564 --- /dev/null +++ b/env/build_wcoss_dell_p3_intel @@ -0,0 +1,31 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA WCOSS Dell machine using Intel-18.0.1.163" +} + +module-whatis "Loads libraries needed for building SRW on WCOSS Dell" + +module purge + +module load ips/18.0.5.274 +module load impi/18.0.1 +module load lsf/10.1 +module load python/3.6.3 + +### hpc-stack ### +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.5.274 +module load hpc-impi/18.0.1 + +module load srw_common + +module load cmake/3.20.0 +module load HPSS/5.0.2.5 + +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform wcoss_dell_p3 diff --git a/env/build_wcoss_dell_p3_intel.env b/env/build_wcoss_dell_p3_intel.env deleted file mode 100644 index 31978ef186..0000000000 --- a/env/build_wcoss_dell_p3_intel.env +++ /dev/null @@ -1,24 +0,0 @@ -#Setup instructions for NOAA WCOSS Dell using Intel-18.0.1.163 (bash shell) - -module purge - -module load ips/18.0.5.274 -module load impi/18.0.1 -module load lsf/10.1 -module load python/3.6.3 - -### hpc-stack ### -module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack -module load hpc/1.1.0 -module load hpc-ips/18.0.5.274 -module load hpc-impi/18.0.1 - -module load srw_common - -module load cmake/3.20.0 -module load HPSS/5.0.2.5 - -export CMAKE_C_COMPILER=mpiicc -export CMAKE_CXX_COMPILER=mpiicpc -export CMAKE_Fortran_COMPILER=mpiifort -export CMAKE_Platform=wcoss_dell_p3 diff --git a/env/wflow_cheyenne b/env/wflow_cheyenne new file mode 100644 index 0000000000..867b0d2aac --- /dev/null +++ b/env/wflow_cheyenne @@ -0,0 +1,16 @@ +#%Module + +proc ModulesHelp { } { + put stderr "This module loads python environement for running SRW on" + put stderr "the CISL machine Cheyenne" +} + +module-whatis "Loads libraries needed for running SRW on Cheyenne" + +module load ncarenv +module use -a /glade/p/ral/jntp/UFS_SRW_app/modules/ +module load rocoto + +if { [module-info mode load] } { + puts "ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/regional_workflow;" +} diff --git a/env/wflow_cheyenne.env b/env/wflow_cheyenne.env deleted file mode 100644 index 1f0ecb485c..0000000000 --- a/env/wflow_cheyenne.env +++ /dev/null @@ -1,6 +0,0 @@ -# Python environment for workflow on Cheyenne - -module load ncarenv -ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/regional_workflow -module use -a /glade/p/ral/jntp/UFS_SRW_app/modules/ -module load rocoto diff --git a/env/wflow_hera b/env/wflow_hera new file mode 100644 index 0000000000..f3d69205f3 --- /dev/null +++ b/env/wflow_hera @@ -0,0 +1,21 @@ +#%Module + +proc ModulesHelp { } { + put stderr "This module loads python environement for running SRW on" + put stderr "the NOAA RDHPC machine Hera" +} + +module-whatis "Loads libraries needed for running SRW on Hera" + +module load rocoto + +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +if { [module-info mode load] } { + puts "conda activate regional_workflow;" +} + +if {[module-info mode remove]} { + puts "conda deactivate;" +} diff --git a/env/wflow_hera.env b/env/wflow_hera.env deleted file mode 100644 index f526c434f5..0000000000 --- a/env/wflow_hera.env +++ /dev/null @@ -1,7 +0,0 @@ -# Python environment for workflow on Hera - -module load rocoto - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -conda activate regional_workflow diff --git a/env/wflow_jet b/env/wflow_jet new file mode 100644 index 0000000000..c845dcb0de --- /dev/null +++ b/env/wflow_jet @@ -0,0 +1,21 @@ +#%Module + +proc ModulesHelp { } { + put stderr "This module loads python environement for running SRW on" + put stderr "the NOAA RDHPC machine Jet" +} + +module-whatis "Loads libraries needed for running SRW on Jet" + +module load rocoto + +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +if { [module-info mode load] } { + puts "conda activate regional_workflow;" +} + +if {[module-info mode remove]} { + puts "conda deactivate;" +} diff --git a/env/wflow_jet.env b/env/wflow_jet.env deleted file mode 100644 index f543bed8f1..0000000000 --- a/env/wflow_jet.env +++ /dev/null @@ -1,7 +0,0 @@ -# Python environment for workflow on Jet - -module load rocoto - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -conda activate regional_workflow diff --git a/env/wflow_odin b/env/wflow_odin new file mode 100644 index 0000000000..6fa86ac2ad --- /dev/null +++ b/env/wflow_odin @@ -0,0 +1,47 @@ +#%Module + +proc ModulesHelp { } { + put stderr "This module loads python environement for running SRW on" + put stderr "the NOAA RDHPC machine Hera" +} + +module-whatis "Loads libraries needed for running SRW on Hera" + +if { [module-info mode load] } { + # >>> conda initialize >>> + # !! Contents within this block are managed by 'conda init' !! + set cmd_init {\ +__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then + . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" + else + export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" + fi +fi +unset __conda_setup + } + puts $cmd_init + # <<< conda initialize <<< + + # To make "regional_workflow" avaiable, + # you should uncomment the following lines, which create file ".condarc" + # or install the environment yourself. + # +# set cmd {\ +#cat > $HOME/.condarc <>> conda initialize >>> -# !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then - . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" - else - export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" - fi -fi -unset __conda_setup -# <<< conda initialize <<< - -# To make "regional_workflow" avaiable, -# you should uncomment the following lines, which create file ".condarc" -# or install the environment yourself. -#cat > $HOME/.condarc < Date: Wed, 20 Apr 2022 22:30:43 +0000 Subject: [PATCH 03/26] Leave conda activation to the user. --- env/wflow_cheyenne | 4 +++- env/wflow_hera | 6 ++---- env/wflow_jet | 6 ++---- env/wflow_odin | 12 +++++------- env/wflow_orion | 6 ++---- env/wflow_singularity | 6 ++---- 6 files changed, 16 insertions(+), 24 deletions(-) diff --git a/env/wflow_cheyenne b/env/wflow_cheyenne index 867b0d2aac..ca940a5387 100644 --- a/env/wflow_cheyenne +++ b/env/wflow_cheyenne @@ -12,5 +12,7 @@ module use -a /glade/p/ral/jntp/UFS_SRW_app/modules/ module load rocoto if { [module-info mode load] } { - puts "ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/regional_workflow;" + puts stderr "Please do the following to activate ncar_pylib: + > ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/regional_workflow" } + diff --git a/env/wflow_hera b/env/wflow_hera index f3d69205f3..724953e4bb 100644 --- a/env/wflow_hera +++ b/env/wflow_hera @@ -13,9 +13,7 @@ module use -a /contrib/miniconda3/modulefiles module load miniconda3 if { [module-info mode load] } { - puts "conda activate regional_workflow;" + puts stderr "Please do the following to activate conda: + > conda activate regional_workflow" } -if {[module-info mode remove]} { - puts "conda deactivate;" -} diff --git a/env/wflow_jet b/env/wflow_jet index c845dcb0de..e340ca4a12 100644 --- a/env/wflow_jet +++ b/env/wflow_jet @@ -13,9 +13,7 @@ module use -a /contrib/miniconda3/modulefiles module load miniconda3 if { [module-info mode load] } { - puts "conda activate regional_workflow;" + puts stderr "Please do the following to activate conda: + > conda activate regional_workflow" } -if {[module-info mode remove]} { - puts "conda deactivate;" -} diff --git a/env/wflow_odin b/env/wflow_odin index 6fa86ac2ad..01cee9e336 100644 --- a/env/wflow_odin +++ b/env/wflow_odin @@ -2,10 +2,10 @@ proc ModulesHelp { } { put stderr "This module loads python environement for running SRW on" - put stderr "the NOAA RDHPC machine Hera" + put stderr "Odin" } -module-whatis "Loads libraries needed for running SRW on Hera" +module-whatis "Loads libraries needed for running SRW on Odin" if { [module-info mode load] } { # >>> conda initialize >>> @@ -38,10 +38,8 @@ unset __conda_setup # } # puts $cmd - puts "conda config --set changeps1 False;" - puts "conda activate regional_workflow;" + puts stderr "Please do the following to activate conda: + > conda config --set changeps1 False + > conda activate regional_workflow" } -if {[module-info mode remove]} { - puts "conda deactivate;" -} diff --git a/env/wflow_orion b/env/wflow_orion index f464869de8..fb3439b0b9 100644 --- a/env/wflow_orion +++ b/env/wflow_orion @@ -13,9 +13,7 @@ module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3/3.8 if { [module-info mode load] } { - puts "conda activate regional_workflow;" + puts stderr "Please do the following to activate conda: + > conda activate regional_workflow" } -if {[module-info mode remove]} { - puts "conda deactivate;" -} diff --git a/env/wflow_singularity b/env/wflow_singularity index a3abbbbbcc..5a9bb7db35 100644 --- a/env/wflow_singularity +++ b/env/wflow_singularity @@ -8,9 +8,7 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for running SRW in a singularity container" if { [module-info mode load] } { - puts "conda activate regional_workflow;" + puts stderr "Please do the following to activate conda: + > conda activate regional_workflow" } -if {[module-info mode remove]} { - puts "conda deactivate;" -} From 3235ef49436b081d2241ed659cb591905433a996 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 20 Apr 2022 22:42:49 +0000 Subject: [PATCH 04/26] Remove set_machine script. --- devbuild.sh | 7 ++++--- env/set_machine.sh | 6 ------ test/build.sh | 6 ++++-- 3 files changed, 8 insertions(+), 11 deletions(-) delete mode 100755 env/set_machine.sh diff --git a/devbuild.sh b/devbuild.sh index 40067f9319..7479229734 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -144,7 +144,8 @@ if [ -z $PLATFORM ] ; then fi # set PLATFORM (MACHINE) -source ${SRC_DIR}/env/set_machine.sh $PLATFORM +MACHINE="${PLATFORM}" +printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 set -eu @@ -168,8 +169,8 @@ if [ "${VERBOSE}" = true ] ; then fi # set ENV_FILE for this platform/compiler combination -ENV_FILE="${SRC_DIR}/env/build_${PLATFORM}_${COMPILER}" -if [ ! -f "${ENV_FILE}" ]; then +ENV_FILE="build_${PLATFORM}_${COMPILER}" +if [ ! -f "${SRC_DIR}/env/${ENV_FILE}" ]; then printf "ERROR: environment file does not exist for platform/compiler\n" >&2 printf " ENV_FILE=${ENV_FILE}\n" >&2 printf " PLATFORM=${PLATFORM}\n" >&2 diff --git a/env/set_machine.sh b/env/set_machine.sh deleted file mode 100755 index 05985d228d..0000000000 --- a/env/set_machine.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -MACHINE="$1" -PLATFORM="${MACHINE}" - -printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 diff --git a/test/build.sh b/test/build.sh index ea3e8a4ae4..95458dbcc1 100755 --- a/test/build.sh +++ b/test/build.sh @@ -34,8 +34,10 @@ TEST_DIR=$( pwd ) # Directory with this script TOP_DIR=${TEST_DIR}/.. # Top level (umbrella repo) directory TEST_OUTPUT=${TEST_DIR}/build_test${PID}.out -# Detect MACHINE -source ${TOP_DIR}/env/set_machine.sh $1 +# set PLATFORM (MACHINE) +MACHINE="$1" +PLATFORM="${MACHINE}" +printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 machine=$(echo "${MACHINE}" | tr '[A-Z]' '[a-z]') # scripts in sorc need lower case machine name From 6e352ec9daa26e2cbf457d29eef2a3dd9e507f95 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 20 Apr 2022 23:22:26 +0000 Subject: [PATCH 05/26] Rename env to modulefiles --- devbuild.sh | 10 +++++----- {env => modulefiles}/build_cheyenne_gnu | 0 {env => modulefiles}/build_cheyenne_intel | 0 {env => modulefiles}/build_hera_intel | 0 {env => modulefiles}/build_jet_intel | 0 {env => modulefiles}/build_macos_gccgfortran | 0 {env => modulefiles}/build_odin_intel | 0 {env => modulefiles}/build_orion_intel | 0 {env => modulefiles}/build_singularity_gnu | 0 {env => modulefiles}/build_wcoss_dell_p3_intel | 0 {env => modulefiles}/srw_common | 0 {env => modulefiles}/wflow_cheyenne | 0 {env => modulefiles}/wflow_hera | 0 {env => modulefiles}/wflow_jet | 0 {env => modulefiles}/wflow_odin | 0 {env => modulefiles}/wflow_orion | 0 {env => modulefiles}/wflow_singularity | 0 {env => modulefiles}/wflow_wcoss_dell_p3 | 0 18 files changed, 5 insertions(+), 5 deletions(-) rename {env => modulefiles}/build_cheyenne_gnu (100%) rename {env => modulefiles}/build_cheyenne_intel (100%) rename {env => modulefiles}/build_hera_intel (100%) rename {env => modulefiles}/build_jet_intel (100%) rename {env => modulefiles}/build_macos_gccgfortran (100%) rename {env => modulefiles}/build_odin_intel (100%) rename {env => modulefiles}/build_orion_intel (100%) rename {env => modulefiles}/build_singularity_gnu (100%) rename {env => modulefiles}/build_wcoss_dell_p3_intel (100%) rename {env => modulefiles}/srw_common (100%) rename {env => modulefiles}/wflow_cheyenne (100%) rename {env => modulefiles}/wflow_hera (100%) rename {env => modulefiles}/wflow_jet (100%) rename {env => modulefiles}/wflow_odin (100%) rename {env => modulefiles}/wflow_orion (100%) rename {env => modulefiles}/wflow_singularity (100%) rename {env => modulefiles}/wflow_wcoss_dell_p3 (100%) diff --git a/devbuild.sh b/devbuild.sh index 7479229734..adf22725b9 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -170,8 +170,8 @@ fi # set ENV_FILE for this platform/compiler combination ENV_FILE="build_${PLATFORM}_${COMPILER}" -if [ ! -f "${SRC_DIR}/env/${ENV_FILE}" ]; then - printf "ERROR: environment file does not exist for platform/compiler\n" >&2 +if [ ! -f "${SRC_DIR}/modulefiles/${ENV_FILE}" ]; then + printf "ERROR: module file does not exist for platform/compiler\n" >&2 printf " ENV_FILE=${ENV_FILE}\n" >&2 printf " PLATFORM=${PLATFORM}\n" >&2 printf " COMPILER=${COMPILER}\n\n" >&2 @@ -237,9 +237,9 @@ if [ "${VERBOSE}" = true ]; then MAKE_SETTINGS="${MAKE_SETTINGS} VERBOSE=1" fi -# source the environment file for this platform/compiler combination, then build the code -printf "... Source ENV_FILE and create BUILD directory ...\n" -module use ${SRC_DIR}/env +# source the module file for this platform/compiler combination, then build the code +printf "... Load MODULE_FILE and create BUILD directory ...\n" +module use ${SRC_DIR}/modulefiles module load ${ENV_FILE} module list mkdir -p ${BUILD_DIR} diff --git a/env/build_cheyenne_gnu b/modulefiles/build_cheyenne_gnu similarity index 100% rename from env/build_cheyenne_gnu rename to modulefiles/build_cheyenne_gnu diff --git a/env/build_cheyenne_intel b/modulefiles/build_cheyenne_intel similarity index 100% rename from env/build_cheyenne_intel rename to modulefiles/build_cheyenne_intel diff --git a/env/build_hera_intel b/modulefiles/build_hera_intel similarity index 100% rename from env/build_hera_intel rename to modulefiles/build_hera_intel diff --git a/env/build_jet_intel b/modulefiles/build_jet_intel similarity index 100% rename from env/build_jet_intel rename to modulefiles/build_jet_intel diff --git a/env/build_macos_gccgfortran b/modulefiles/build_macos_gccgfortran similarity index 100% rename from env/build_macos_gccgfortran rename to modulefiles/build_macos_gccgfortran diff --git a/env/build_odin_intel b/modulefiles/build_odin_intel similarity index 100% rename from env/build_odin_intel rename to modulefiles/build_odin_intel diff --git a/env/build_orion_intel b/modulefiles/build_orion_intel similarity index 100% rename from env/build_orion_intel rename to modulefiles/build_orion_intel diff --git a/env/build_singularity_gnu b/modulefiles/build_singularity_gnu similarity index 100% rename from env/build_singularity_gnu rename to modulefiles/build_singularity_gnu diff --git a/env/build_wcoss_dell_p3_intel b/modulefiles/build_wcoss_dell_p3_intel similarity index 100% rename from env/build_wcoss_dell_p3_intel rename to modulefiles/build_wcoss_dell_p3_intel diff --git a/env/srw_common b/modulefiles/srw_common similarity index 100% rename from env/srw_common rename to modulefiles/srw_common diff --git a/env/wflow_cheyenne b/modulefiles/wflow_cheyenne similarity index 100% rename from env/wflow_cheyenne rename to modulefiles/wflow_cheyenne diff --git a/env/wflow_hera b/modulefiles/wflow_hera similarity index 100% rename from env/wflow_hera rename to modulefiles/wflow_hera diff --git a/env/wflow_jet b/modulefiles/wflow_jet similarity index 100% rename from env/wflow_jet rename to modulefiles/wflow_jet diff --git a/env/wflow_odin b/modulefiles/wflow_odin similarity index 100% rename from env/wflow_odin rename to modulefiles/wflow_odin diff --git a/env/wflow_orion b/modulefiles/wflow_orion similarity index 100% rename from env/wflow_orion rename to modulefiles/wflow_orion diff --git a/env/wflow_singularity b/modulefiles/wflow_singularity similarity index 100% rename from env/wflow_singularity rename to modulefiles/wflow_singularity diff --git a/env/wflow_wcoss_dell_p3 b/modulefiles/wflow_wcoss_dell_p3 similarity index 100% rename from env/wflow_wcoss_dell_p3 rename to modulefiles/wflow_wcoss_dell_p3 From 9e5d2ad4f0769c3536aae0d729ed3f315f860eeb Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 25 Apr 2022 22:02:26 +0000 Subject: [PATCH 06/26] Minor fix. --- modulefiles/wflow_odin | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modulefiles/wflow_odin b/modulefiles/wflow_odin index 01cee9e336..cfd56e92c1 100644 --- a/modulefiles/wflow_odin +++ b/modulefiles/wflow_odin @@ -10,6 +10,7 @@ module-whatis "Loads libraries needed for running SRW on Odin" if { [module-info mode load] } { # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! + set cmd_init {\ __conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then @@ -23,7 +24,8 @@ else fi unset __conda_setup } - puts $cmd_init + + system "$cmd_init" # <<< conda initialize <<< # To make "regional_workflow" avaiable, From 27e517456ce081b0ec13d84891bddbbcc0ba4f4c Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 27 Apr 2022 17:07:25 +0000 Subject: [PATCH 07/26] Minor fix --- modulefiles/build_macos_gnu | 4 ++-- modulefiles/build_odin_intel | 8 +++++--- modulefiles/wflow_macos | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 0bd0249ef4..11fb6ba487 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -17,8 +17,8 @@ setenv BASH_ENV "/opt/homebrew/opt/lmod/init/bash" #setenv BASH_ENV "/usr/local/opt/lmod/init/bash" if { [module-info mode load] } { - system "source $env(BASH_ENV)" - system "ulimit -S -s unlimited" + system "source $env(BASH_ENV);" + system "ulimit -S -s unlimited;" } # This path should point to your HPCstack installation directory diff --git a/modulefiles/build_odin_intel b/modulefiles/build_odin_intel index 164b0b32d3..6692eec9f5 100644 --- a/modulefiles/build_odin_intel +++ b/modulefiles/build_odin_intel @@ -7,11 +7,13 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on odin" -module unload modules -unset -f module +if { [module-info mode load] } { + module unload modules + system "unset -f module;" +} setenv BASH_ENV "/usr/local/lmod/8.3.1/init/bash" -puts "source $BASH_ENV;" +puts "source $env(BASH_ENV);" setenv LMOD_SYSTEM_DEFAULT_MODULES "PrgEnv-intel:cray-mpich:intel:craype" module --initial_load --no_redirect restore #module use <$HOME>/ diff --git a/modulefiles/wflow_macos b/modulefiles/wflow_macos index 3873b36d8f..dcb0106c2a 100644 --- a/modulefiles/wflow_macos +++ b/modulefiles/wflow_macos @@ -14,7 +14,7 @@ setenv CMAKE_Platform macos # setenv VENV "/Users/username/venv/regional_workflow" if { [module-info mode load] } { - system "source $env(VENV)/bin/activate" + system "source $env(VENV)/bin/activate;" } # Uncomment if Rocoto workflow manager is used From 5a72fdc2da1754337441d7dac329386ecea38f86 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 27 Apr 2022 21:39:18 +0000 Subject: [PATCH 08/26] Take out *module purge* from modufiles and put it in devbuild.sh --- devbuild.sh | 13 +++++++------ modulefiles/build_cheyenne_gnu | 2 -- modulefiles/build_cheyenne_intel | 2 -- modulefiles/build_hera_intel | 2 -- modulefiles/build_jet_intel | 2 -- modulefiles/build_macos_gnu | 1 - modulefiles/build_orion_intel | 2 -- modulefiles/build_singularity_gnu | 2 -- modulefiles/build_wcoss_dell_p3_intel | 2 -- 9 files changed, 7 insertions(+), 21 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index adf22725b9..58e715fca6 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -168,18 +168,18 @@ if [ "${VERBOSE}" = true ] ; then settings fi -# set ENV_FILE for this platform/compiler combination -ENV_FILE="build_${PLATFORM}_${COMPILER}" -if [ ! -f "${SRC_DIR}/modulefiles/${ENV_FILE}" ]; then +# set MODULE_FILE for this platform/compiler combination +MODULE_FILE="build_${PLATFORM}_${COMPILER}" +if [ ! -f "${SRC_DIR}/modulefiles/${MODULE_FILE}" ]; then printf "ERROR: module file does not exist for platform/compiler\n" >&2 - printf " ENV_FILE=${ENV_FILE}\n" >&2 + printf " MODULE_FILE=${MODULE_FILE}\n" >&2 printf " PLATFORM=${PLATFORM}\n" >&2 printf " COMPILER=${COMPILER}\n\n" >&2 usage >&2 exit 64 fi -printf "ENV_FILE=${ENV_FILE}\n" >&2 +printf "MODULE_FILE=${MODULE_FILE}\n" >&2 # if build directory already exists then exit if [ "${CLEAN}" = true ]; then @@ -240,7 +240,8 @@ fi # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" module use ${SRC_DIR}/modulefiles -module load ${ENV_FILE} +module purge +module load ${MODULE_FILE} module list mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} diff --git a/modulefiles/build_cheyenne_gnu b/modulefiles/build_cheyenne_gnu index f6a1111579..a5730c88b6 100644 --- a/modulefiles/build_cheyenne_gnu +++ b/modulefiles/build_cheyenne_gnu @@ -7,8 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Cheyenne" -module purge - module load cmake/3.22.0 module load ncarenv/1.3 module load gnu/10.1.0 diff --git a/modulefiles/build_cheyenne_intel b/modulefiles/build_cheyenne_intel index bfc2c16e23..fc4a6392c5 100644 --- a/modulefiles/build_cheyenne_intel +++ b/modulefiles/build_cheyenne_intel @@ -7,8 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Cheyenne" -module purge - module load cmake/3.22.0 module load ncarenv/1.3 module load intel/2021.2 diff --git a/modulefiles/build_hera_intel b/modulefiles/build_hera_intel index 76c317c3a0..521a93e99d 100644 --- a/modulefiles/build_hera_intel +++ b/modulefiles/build_hera_intel @@ -7,8 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Hera" -module purge - module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles module load miniconda3/3.7.3 diff --git a/modulefiles/build_jet_intel b/modulefiles/build_jet_intel index 405999038a..59b192cfed 100644 --- a/modulefiles/build_jet_intel +++ b/modulefiles/build_jet_intel @@ -7,8 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on JET" -module purge - module use /contrib/sutils/modulefiles module load sutils diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 11fb6ba487..42896a463a 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -27,7 +27,6 @@ setenv HPCstack "/Users/username/hpc-stack/install" # This path should point to your SRW Application directory setenv SRW "/Users/username/ufs-srweather-app" -module purge # Load HPC stack module use ${HPCstack}/modulefiles/stack module load hpc diff --git a/modulefiles/build_orion_intel b/modulefiles/build_orion_intel index 7dc4888883..3be5e3fc1a 100644 --- a/modulefiles/build_orion_intel +++ b/modulefiles/build_orion_intel @@ -7,8 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Orion" -module purge - module load contrib noaatools module load cmake/3.22.1 diff --git a/modulefiles/build_singularity_gnu b/modulefiles/build_singularity_gnu index ee58e550be..2ee2baef4a 100644 --- a/modulefiles/build_singularity_gnu +++ b/modulefiles/build_singularity_gnu @@ -9,8 +9,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW in a singularity container" -module purge - module use /opt/hpc-modules/modulefiles/stack module load hpc diff --git a/modulefiles/build_wcoss_dell_p3_intel b/modulefiles/build_wcoss_dell_p3_intel index 7668396564..ea752a35cf 100644 --- a/modulefiles/build_wcoss_dell_p3_intel +++ b/modulefiles/build_wcoss_dell_p3_intel @@ -7,8 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on WCOSS Dell" -module purge - module load ips/18.0.5.274 module load impi/18.0.1 module load lsf/10.1 From 5e3a5256f8cbcebc97bd5b597f1ab3bb82dfaff9 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 27 Apr 2022 23:27:41 +0000 Subject: [PATCH 09/26] Activate conda directly in signularity modulefile. --- modulefiles/build_macos_gnu | 4 ++-- modulefiles/wflow_singularity | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 42896a463a..6e6bdc8fb3 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -28,7 +28,7 @@ setenv HPCstack "/Users/username/hpc-stack/install" setenv SRW "/Users/username/ufs-srweather-app" # Load HPC stack -module use ${HPCstack}/modulefiles/stack +module use $env(HPCstack)/modulefiles/stack module load hpc module load hpc-python # @@ -36,7 +36,7 @@ module load hpc-gnu module load openmpi module load hpc-openmpi -module use ${SRW}/env +module use $env(SRW)/env #module load srw_common module load jasper/2.0.25 module load zlib/1.2.11 diff --git a/modulefiles/wflow_singularity b/modulefiles/wflow_singularity index 5a9bb7db35..72d5b22973 100644 --- a/modulefiles/wflow_singularity +++ b/modulefiles/wflow_singularity @@ -8,7 +8,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for running SRW in a singularity container" if { [module-info mode load] } { - puts stderr "Please do the following to activate conda: - > conda activate regional_workflow" + puts "conda activate regional_workflow;" } From e4a8c8c64190ba82e34852bc32493f78d601004f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 28 Apr 2022 02:03:33 +0000 Subject: [PATCH 10/26] Minor fixes. --- modulefiles/build_macos_gnu | 2 +- modulefiles/wflow_macos | 2 +- modulefiles/wflow_odin | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 6e6bdc8fb3..85d7291208 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -36,7 +36,7 @@ module load hpc-gnu module load openmpi module load hpc-openmpi -module use $env(SRW)/env +module use $env(SRW)/modulefiles #module load srw_common module load jasper/2.0.25 module load zlib/1.2.11 diff --git a/modulefiles/wflow_macos b/modulefiles/wflow_macos index dcb0106c2a..7481b36911 100644 --- a/modulefiles/wflow_macos +++ b/modulefiles/wflow_macos @@ -19,7 +19,7 @@ if { [module-info mode load] } { # Uncomment if Rocoto workflow manager is used # This path should point to your Rocoto module location -# ROCOTOmod=/Users/username/modules +# set ROCOTOmod "/Users/username/modules" # module use ${ROCOTOmod} # module load rocoto # module list diff --git a/modulefiles/wflow_odin b/modulefiles/wflow_odin index cfd56e92c1..42a3626e22 100644 --- a/modulefiles/wflow_odin +++ b/modulefiles/wflow_odin @@ -38,7 +38,7 @@ unset __conda_setup # - /home/yunheng.wang/.conda/envs #EOF # } -# puts $cmd +# system "$cmd" puts stderr "Please do the following to activate conda: > conda config --set changeps1 False From 2189a2f4862b7edd7c6255ce03db7e1dc6620b10 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 28 Apr 2022 17:58:08 +0000 Subject: [PATCH 11/26] Add Gaea modulefiles. --- modulefiles/build_gaea_intel | 28 ++++++++++++++++++++++++++++ modulefiles/build_gaea_intel.env | 18 ------------------ modulefiles/wflow_gaea | 15 +++++++++++++++ modulefiles/wflow_gaea.env | 8 -------- 4 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 modulefiles/build_gaea_intel delete mode 100644 modulefiles/build_gaea_intel.env create mode 100644 modulefiles/wflow_gaea delete mode 100644 modulefiles/wflow_gaea.env diff --git a/modulefiles/build_gaea_intel b/modulefiles/build_gaea_intel new file mode 100644 index 0000000000..56208e3ad3 --- /dev/null +++ b/modulefiles/build_gaea_intel @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA RDHPC machine Gaea using Intel-18.0.6.288" +} + +module-whatis "Loads libraries needed for building SRW on Gaea" + +if { [module-info mode load] } { + system "source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh;" +} +module use /lustre/f2/pdata/ncep_shared/hpc-stack/modulefiles/stack +module load hpc hpc-intel hpc-cray-mpich +module load srw_common + +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load cmake/3.20.1 + +setenv CC cc +setenv FC ftn +setenv CXX CC +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform gaea.intel + diff --git a/modulefiles/build_gaea_intel.env b/modulefiles/build_gaea_intel.env deleted file mode 100644 index f7ac329df1..0000000000 --- a/modulefiles/build_gaea_intel.env +++ /dev/null @@ -1,18 +0,0 @@ -#Setup instructions for NOAA RDHPC Gaea using Intel-18.0.6.288 (bash shell) -source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh -module use /lustre/f2/pdata/ncep_shared/hpc-stack/modulefiles/stack -module load hpc hpc-intel hpc-cray-mpich -module load srw_common - -module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles -module load rocoto -module load cmake/3.20.1 - -export CC=cc -export FC=ftn -export CXX=CC -export CMAKE_C_COMPILER=cc -export CMAKE_CXX_COMPILER=CC -export CMAKE_Fortran_COMPILER=ftn -export CMAKE_Platform=gaea.intel - diff --git a/modulefiles/wflow_gaea b/modulefiles/wflow_gaea new file mode 100644 index 0000000000..c9889a4403 --- /dev/null +++ b/modulefiles/wflow_gaea @@ -0,0 +1,15 @@ +#%Module + +proc ModulesHelp { } { + put stderr "This module loads python environement for running SRW on" + put stderr "the NOAA RDHPC machine Gaea" +} + +module-whatis "Loads libraries needed for running SRW on Gaea" + +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv CONDA_DEFAULT_ENV "regional_workflow" + diff --git a/modulefiles/wflow_gaea.env b/modulefiles/wflow_gaea.env deleted file mode 100644 index 80d336b0f4..0000000000 --- a/modulefiles/wflow_gaea.env +++ /dev/null @@ -1,8 +0,0 @@ -#Setup instructions for NOAA RDHPC Hera using Intel-18.0.5.274 (bash shell) - -module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles -module load rocoto -module load miniconda3 - -export CONDA_DEFAULT_ENV=regional_workflow - From c53859ac371ec50e9188c1de9ee080baa400b87f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 01:30:08 +0000 Subject: [PATCH 12/26] Restore odin env files. --- env/build_odin_intel.env | 58 +++++++++++++++++++++++++++++++ env/wflow_odin.env | 25 ++++++++++++++ modulefiles/build_odin_intel | 67 ------------------------------------ modulefiles/wflow_odin | 47 ------------------------- 4 files changed, 83 insertions(+), 114 deletions(-) create mode 100644 env/build_odin_intel.env create mode 100644 env/wflow_odin.env delete mode 100644 modulefiles/build_odin_intel delete mode 100644 modulefiles/wflow_odin diff --git a/env/build_odin_intel.env b/env/build_odin_intel.env new file mode 100644 index 0000000000..cf69d3d091 --- /dev/null +++ b/env/build_odin_intel.env @@ -0,0 +1,58 @@ +#Setup instructions for NOAA RDHPC Jet using Intel-18.0.5.274 (bash shell) + +module unload modules +unset -f module + +export BASH_ENV=/usr/local/lmod/8.3.1/init/bash +source $BASH_ENV +export LMOD_SYSTEM_DEFAULT_MODULES=PrgEnv-intel:cray-mpich:intel:craype +module --initial_load --no_redirect restore +#module use <$HOME>/ +export MODULEPATH=/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles + +#module purge +export CMAKE=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake +export PATH=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH} + +module load hpc/1.2.0 +module load hpc-intel +module load hpc-cray-mpich + +#module load srw_common + +module load jasper +module load zlib +module load png + +#module load cray-hdf5 +#module load cray-netcdf +module load esmf +module load fms + +module load bacio +module load crtm +module load g2 +module load g2tmpl +module load ip +module load sp +module load w3nco +module load upp + +module load gftl-shared +module load yafyaml +module load mapl + +module load gfsio +module load landsfcutil +module load nemsio +module load nemsiogfs +module load sfcio +module load sigio +module load w3emc +module load wgrib2 + +export CMAKE_C_COMPILER=cc +export CMAKE_CXX_COMPILER=CC +export CMAKE_Fortran_COMPILER=ftn +export CMAKE_Platform=odin.intel + diff --git a/env/wflow_odin.env b/env/wflow_odin.env new file mode 100644 index 0000000000..2e3f60e4ea --- /dev/null +++ b/env/wflow_odin.env @@ -0,0 +1,25 @@ +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then + . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" + else + export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" + fi +fi +unset __conda_setup +# <<< conda initialize <<< + +# To make "regional_workflow" avaiable, +# you should uncomment the following lines, which create file ".condarc" +# or install the environment yourself. +#cat > $HOME/.condarc </ -setenv MODULEPATH "/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" - -#module purge -setenv CMAKE "/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake" -setenv PATH "/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH}" - -module load hpc/1.2.0 -module load hpc-intel -module load hpc-cray-mpich - -#module load srw_common - -module load jasper -module load zlib -module load png - -#module load cray-hdf5 -#module load cray-netcdf -module load esmf -module load fms - -module load bacio -module load crtm -module load g2 -module load g2tmpl -module load ip -module load sp -module load w3nco -module load upp - -module load gftl-shared -module load yafyaml -module load mapl - -module load gfsio -module load landsfcutil -module load nemsio -module load nemsiogfs -module load sfcio -module load sigio -module load w3emc -module load wgrib2 - -setenv CMAKE_C_COMPILER cc -setenv CMAKE_CXX_COMPILER CC -setenv CMAKE_Fortran_COMPILER ftn -setenv CMAKE_Platform odin.intel - diff --git a/modulefiles/wflow_odin b/modulefiles/wflow_odin deleted file mode 100644 index 42a3626e22..0000000000 --- a/modulefiles/wflow_odin +++ /dev/null @@ -1,47 +0,0 @@ -#%Module - -proc ModulesHelp { } { - put stderr "This module loads python environement for running SRW on" - put stderr "Odin" -} - -module-whatis "Loads libraries needed for running SRW on Odin" - -if { [module-info mode load] } { - # >>> conda initialize >>> - # !! Contents within this block are managed by 'conda init' !! - - set cmd_init {\ -__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then - . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" - else - export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" - fi -fi -unset __conda_setup - } - - system "$cmd_init" - # <<< conda initialize <<< - - # To make "regional_workflow" avaiable, - # you should uncomment the following lines, which create file ".condarc" - # or install the environment yourself. - # -# set cmd {\ -#cat > $HOME/.condarc < conda config --set changeps1 False - > conda activate regional_workflow" -} - From c7c297004a93df7d97c760fc4b9711ee32916401 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 04:12:59 +0000 Subject: [PATCH 13/26] Bug fixes in singularity modulefiles. --- modulefiles/build_singularity_gnu | 4 ++-- modulefiles/wflow_singularity | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modulefiles/build_singularity_gnu b/modulefiles/build_singularity_gnu index 2ee2baef4a..9a28fc1ab1 100644 --- a/modulefiles/build_singularity_gnu +++ b/modulefiles/build_singularity_gnu @@ -32,8 +32,8 @@ module load g2tmpl module load ip module load sp module load w3nco -module load cmake/3.21.1 -module load module load gfsio +module load cmake +module load gfsio module load wgrib2 module load upp diff --git a/modulefiles/wflow_singularity b/modulefiles/wflow_singularity index 72d5b22973..af765e015b 100644 --- a/modulefiles/wflow_singularity +++ b/modulefiles/wflow_singularity @@ -7,7 +7,10 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for running SRW in a singularity container" +module use -a /opt/hpc-modules/modulefiles/core +module load miniconda3 + if { [module-info mode load] } { - puts "conda activate regional_workflow;" + system "conda activate regional_workflow;" } From 569d32a26c6ceee2264928798889902a4a745c89 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 19:18:30 +0000 Subject: [PATCH 14/26] Move activation of Lmod to devbuild.sh --- devbuild.sh | 18 ++++++++++++++++-- modulefiles/build_gaea_intel | 3 --- modulefiles/build_macos_gnu | 10 ---------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index 58e715fca6..e8b9aae002 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -152,11 +152,11 @@ set -eu # automatically determine compiler if [ -z "${COMPILER}" ] ; then case ${PLATFORM} in - jet|hera) COMPILER=intel ;; + jet|hera|gaea) COMPILER=intel ;; orion) COMPILER=intel ;; wcoss_dell_p3) COMPILER=intel ;; cheyenne) COMPILER=intel ;; - macos) COMPILER=gccgfortran ;; + macos) COMPILER=gnu ;; *) printf "ERROR: Unknown platform ${PLATFORM}\n" >&2; usage >&2; exit 1 ;; esac fi @@ -237,6 +237,20 @@ if [ "${VERBOSE}" = true ]; then MAKE_SETTINGS="${MAKE_SETTINGS} VERBOSE=1" fi +# Before we go on load modules, we first need to activate Lmod for some systems +if [ "$MACHINE" = macos ]; then + # Choose lmod install location + export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" + # export BASH_ENV="/usr/local/opt/lmod/init/bash" + + source $BASH_ENV +elif [ "$MACHINE" = gaea ]; then + + source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh +elif [ "$MACHINE" = odin ]; then + : +fi + # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" module use ${SRC_DIR}/modulefiles diff --git a/modulefiles/build_gaea_intel b/modulefiles/build_gaea_intel index 56208e3ad3..7a0accfc87 100644 --- a/modulefiles/build_gaea_intel +++ b/modulefiles/build_gaea_intel @@ -7,9 +7,6 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Gaea" -if { [module-info mode load] } { - system "source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh;" -} module use /lustre/f2/pdata/ncep_shared/hpc-stack/modulefiles/stack module load hpc hpc-intel hpc-cray-mpich module load srw_common diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 85d7291208..7947b2a343 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -7,17 +7,7 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Hera" -# Option 1 (default) : M1 chip, arm64, running natively (not using Rosetta) -# Option 2 (uncomment if needed) Intel chip, x86_64 -# -# Option 1 (default): -setenv BASH_ENV "/opt/homebrew/opt/lmod/init/bash" - -# Option 2 (uncomment if needed): -#setenv BASH_ENV "/usr/local/opt/lmod/init/bash" - if { [module-info mode load] } { - system "source $env(BASH_ENV);" system "ulimit -S -s unlimited;" } From f1583e88989ec21b6f64013b3df6179a628b59b3 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 19:54:17 +0000 Subject: [PATCH 15/26] Don't do 'module purge' on cray systems --- devbuild.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index e8b9aae002..31df0f3482 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -242,19 +242,18 @@ if [ "$MACHINE" = macos ]; then # Choose lmod install location export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" # export BASH_ENV="/usr/local/opt/lmod/init/bash" - source $BASH_ENV elif [ "$MACHINE" = gaea ]; then - - source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh + source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" elif [ "$MACHINE" = odin ]; then : +else: + module purge fi # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" module use ${SRC_DIR}/modulefiles -module purge module load ${MODULE_FILE} module list mkdir -p ${BUILD_DIR} From 8be4258052c3677b11a4a85dc69c781954f5a6e6 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 21:25:20 +0000 Subject: [PATCH 16/26] Put Lmod initialization code in separate script. --- devbuild.sh | 13 +------------ lmod-setup.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 lmod-setup.sh diff --git a/devbuild.sh b/devbuild.sh index 31df0f3482..f18e827e9b 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -238,18 +238,7 @@ if [ "${VERBOSE}" = true ]; then fi # Before we go on load modules, we first need to activate Lmod for some systems -if [ "$MACHINE" = macos ]; then - # Choose lmod install location - export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" - # export BASH_ENV="/usr/local/opt/lmod/init/bash" - source $BASH_ENV -elif [ "$MACHINE" = gaea ]; then - source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" -elif [ "$MACHINE" = odin ]; then - : -else: - module purge -fi +source ${SRC_DIR}/lmod-setup.sh # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" diff --git a/lmod-setup.sh b/lmod-setup.sh new file mode 100644 index 0000000000..9f606b3d94 --- /dev/null +++ b/lmod-setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +if [ "$MACHINE" = macos ]; then + # Choose lmod install location + export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" + # export BASH_ENV="/usr/local/opt/lmod/init/bash" + source $BASH_ENV +elif [ "$MACHINE" = gaea ]; then + source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" +elif [ "$MACHINE" = odin ]; then + module unload modules + unset -f module + + export BASH_ENV=/usr/local/lmod/8.3.1/init/bash + source $BASH_ENV + export LMOD_SYSTEM_DEFAULT_MODULES=PrgEnv-intel:cray-mpich:intel:craype + module --initial_load --no_redirect restore + #module use <$HOME>/ + export MODULEPATH=/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles + + #module purge + export CMAKE=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake + export PATH=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH} +else: + module purge +fi + From 7bb0487c1336f2cf5c9952ea9b43d5f85ba69ae5 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 21:47:21 +0000 Subject: [PATCH 17/26] Go back to using modulefile for odin. --- env/build_odin_intel.env | 58 ------------------------------------ env/wflow_odin.env | 25 ---------------- modulefiles/build_odin_intel | 51 +++++++++++++++++++++++++++++++ modulefiles/wflow_odin | 47 +++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 83 deletions(-) delete mode 100644 env/build_odin_intel.env delete mode 100644 env/wflow_odin.env create mode 100644 modulefiles/build_odin_intel create mode 100644 modulefiles/wflow_odin diff --git a/env/build_odin_intel.env b/env/build_odin_intel.env deleted file mode 100644 index cf69d3d091..0000000000 --- a/env/build_odin_intel.env +++ /dev/null @@ -1,58 +0,0 @@ -#Setup instructions for NOAA RDHPC Jet using Intel-18.0.5.274 (bash shell) - -module unload modules -unset -f module - -export BASH_ENV=/usr/local/lmod/8.3.1/init/bash -source $BASH_ENV -export LMOD_SYSTEM_DEFAULT_MODULES=PrgEnv-intel:cray-mpich:intel:craype -module --initial_load --no_redirect restore -#module use <$HOME>/ -export MODULEPATH=/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles - -#module purge -export CMAKE=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake -export PATH=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH} - -module load hpc/1.2.0 -module load hpc-intel -module load hpc-cray-mpich - -#module load srw_common - -module load jasper -module load zlib -module load png - -#module load cray-hdf5 -#module load cray-netcdf -module load esmf -module load fms - -module load bacio -module load crtm -module load g2 -module load g2tmpl -module load ip -module load sp -module load w3nco -module load upp - -module load gftl-shared -module load yafyaml -module load mapl - -module load gfsio -module load landsfcutil -module load nemsio -module load nemsiogfs -module load sfcio -module load sigio -module load w3emc -module load wgrib2 - -export CMAKE_C_COMPILER=cc -export CMAKE_CXX_COMPILER=CC -export CMAKE_Fortran_COMPILER=ftn -export CMAKE_Platform=odin.intel - diff --git a/env/wflow_odin.env b/env/wflow_odin.env deleted file mode 100644 index 2e3f60e4ea..0000000000 --- a/env/wflow_odin.env +++ /dev/null @@ -1,25 +0,0 @@ -# >>> conda initialize >>> -# !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then - . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" - else - export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" - fi -fi -unset __conda_setup -# <<< conda initialize <<< - -# To make "regional_workflow" avaiable, -# you should uncomment the following lines, which create file ".condarc" -# or install the environment yourself. -#cat > $HOME/.condarc <>> conda initialize >>> + # !! Contents within this block are managed by 'conda init' !! + + set cmd_init {\ +__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then + . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" + else + export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" + fi +fi +unset __conda_setup + } + + system "$cmd_init" + # <<< conda initialize <<< + + # To make "regional_workflow" avaiable, + # you should uncomment the following lines, which create file ".condarc" + # or install the environment yourself. + # +# set cmd {\ +#cat > $HOME/.condarc < conda config --set changeps1 False + > conda activate regional_workflow" +} + From ed4447134eeefe620aabf40b3e4cc16943096d90 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 29 Apr 2022 22:55:48 +0000 Subject: [PATCH 18/26] Optionally pass machine name to lmod-setup.sh --- lmod-setup.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lmod-setup.sh b/lmod-setup.sh index 9f606b3d94..bc3bdb2c06 100644 --- a/lmod-setup.sh +++ b/lmod-setup.sh @@ -1,13 +1,15 @@ #!/bin/bash -if [ "$MACHINE" = macos ]; then +L_MACHINE=${MACHINE:-$1} + +if [ "$L_MACHINE" = macos ]; then # Choose lmod install location export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" # export BASH_ENV="/usr/local/opt/lmod/init/bash" source $BASH_ENV -elif [ "$MACHINE" = gaea ]; then +elif [ "$L_MACHINE" = gaea ]; then source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" -elif [ "$MACHINE" = odin ]; then +elif [ "$L_MACHINE" = odin ]; then module unload modules unset -f module @@ -21,7 +23,7 @@ elif [ "$MACHINE" = odin ]; then #module purge export CMAKE=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake export PATH=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH} -else: +else module purge fi From b1bf8f9900b1b9422ae5aa61a491a47b4c83043f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 30 Apr 2022 03:28:55 +0000 Subject: [PATCH 19/26] Modify odin wflow modulefile. --- modulefiles/build_odin_intel | 2 +- modulefiles/wflow_odin | 30 ++++++++++-------------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/modulefiles/build_odin_intel b/modulefiles/build_odin_intel index 623e0bc0f8..06899da47e 100644 --- a/modulefiles/build_odin_intel +++ b/modulefiles/build_odin_intel @@ -2,7 +2,7 @@ proc ModulesHelp { } { puts stderr "This module loads libraries for building SRW on" - puts stderr "the NOAA RDHPC machine odin using Intel" + puts stderr "the NSSL machine odin using Intel" } module-whatis "Loads libraries needed for building SRW on odin" diff --git a/modulefiles/wflow_odin b/modulefiles/wflow_odin index 42a3626e22..6c84077fe2 100644 --- a/modulefiles/wflow_odin +++ b/modulefiles/wflow_odin @@ -2,7 +2,7 @@ proc ModulesHelp { } { put stderr "This module loads python environement for running SRW on" - put stderr "Odin" + put stderr "NSSL machine Odin" } module-whatis "Loads libraries needed for running SRW on Odin" @@ -10,34 +10,24 @@ module-whatis "Loads libraries needed for running SRW on Odin" if { [module-info mode load] } { # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! - - set cmd_init {\ -__conda_setup="$('/scratch/software/Odin/python/anaconda2/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]; then - . "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" - else - export PATH="/scratch/software/Odin/python/anaconda2/bin:$PATH" - fi -fi -unset __conda_setup + if {[ system "/scratch/software/Odin/python/anaconda2/bin/conda shell.bash hook 2> /dev/null" ]} { + } else { + if {[ file exists "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]} { + system "source /scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh;" + } else { + prepend-path PATH "/scratch/software/Odin/python/anaconda2/bin" + } } - - system "$cmd_init" # <<< conda initialize <<< # To make "regional_workflow" avaiable, # you should uncomment the following lines, which create file ".condarc" # or install the environment yourself. # -# set cmd {\ -#cat > $HOME/.condarc < $HOME/.condarc < Date: Sat, 30 Apr 2022 09:58:30 +0000 Subject: [PATCH 20/26] Allow unknown platforms in devbuild.sh --- devbuild.sh | 9 +++++++-- test/build.sh | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index f18e827e9b..70c9aee5b4 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -156,8 +156,12 @@ if [ -z "${COMPILER}" ] ; then orion) COMPILER=intel ;; wcoss_dell_p3) COMPILER=intel ;; cheyenne) COMPILER=intel ;; - macos) COMPILER=gnu ;; - *) printf "ERROR: Unknown platform ${PLATFORM}\n" >&2; usage >&2; exit 1 ;; + macos,singularity) COMPILER=gnu ;; + odin) COMPILER=intel ;; + *) + COMPILER=intel + printf "WARNING: Setting default COMPILER=intel for new platform ${PLATFORM}\n" >&2; + ;; esac fi @@ -175,6 +179,7 @@ if [ ! -f "${SRC_DIR}/modulefiles/${MODULE_FILE}" ]; then printf " MODULE_FILE=${MODULE_FILE}\n" >&2 printf " PLATFORM=${PLATFORM}\n" >&2 printf " COMPILER=${COMPILER}\n\n" >&2 + printf "Please make sure PLATFORM and COMPILER are set correctly\n" >&2 usage >&2 exit 64 fi diff --git a/test/build.sh b/test/build.sh index 95458dbcc1..83c30910e7 100755 --- a/test/build.sh +++ b/test/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne orion wcoss_cray wcoss_dell_p3 ) +machines=( hera jet cheyenne orion wcoss_cray wcoss_dell_p3 gaea odin singularity ) [[ $# -eq 2 ]] && usage @@ -56,6 +56,8 @@ fi #----------------------------------------------------------------------- if [ "${machine}" == "cheyenne" ] ; then compilers=( intel gnu ) +elif [ "${machine}" == "macos" ] || [ "${machine}" == "singularity" ] ; then + compilers=( gnu ) else compilers=( intel ) fi From 25c0ea95c89853f2bca65cf85ce8d2ae31649cf6 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 30 Apr 2022 12:55:18 +0000 Subject: [PATCH 21/26] Update documentation. --- docs/INSTALL | 33 +++++++++++++++++++++++++++++---- docs/RUNTIME | 19 ++++++++++++++----- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/docs/INSTALL b/docs/INSTALL index 5923285be0..5b18d40845 100644 --- a/docs/INSTALL +++ b/docs/INSTALL @@ -12,10 +12,35 @@ git clone https://github.com/ufs-community/ufs-srweather-app.git cd ufs-srweather-app/ ./manage_externals/checkout_externals -# Prior to building, you must set up the environment so cmake can find the appropriate compilers -# and libraries. For instructions specific to supported platforms, see the "build_[machine]_[compiler].env -# files in the "env" directory. These files give instructions assuming a bash or ksh login shell, for -# csh and tcsh users you will have to modify the commands for setting envronment variables. +# We can build ufs-sreweather-app binaries in two ways. + +# Method 1 +# ======== + +# This is the simplest way to build the binaries + +./devbuild.sh --platform=PLATFORM + +# If compiler auto-detection fails, specify it using + +./devbuild.sh --platform=PLATFORM --compiler=COMPILER + +# Method 2 +# ======== + +# The above instructions will work atleast on Tier-1 systems, if not on all supported machines. +# However, if it fails for some reason, we can build directly with cmake. +# First we need to make sure that there is a modulefile "build_[PLATFORM]_[COMPILER]" in the +# "modulefiles" directory. Also, on some systems (e.g. Gaea/Odin) that come with cray module system, +# we need to swap it for Lmod instead. Assuming the login shell is bash + +source ./lmod-setup.sh PLATFORM + +# From here on, we can assume Lmod is loaded and ready to go. Then we load the specific +# module for a given PLATFORM and COMPILER as follows + +module use modulefiles +module load build_[PLATFORM]_[COMPILER] # Supported CMake flags: # -DCMAKE_INSTALL_PREFIX Location where the bin/ include/ lib/ and share/ directories containing diff --git a/docs/RUNTIME b/docs/RUNTIME index a80a65228e..e2ca78894d 100644 --- a/docs/RUNTIME +++ b/docs/RUNTIME @@ -1,13 +1,22 @@ # Users should load the appropriate python environment for the workflow. # The workflow requires Python 3, with the packages 'PyYAML', 'Jinja2', and 'f90nml' available. -# For users' convenience, the python environment for the workflow is put in 'ufs-srweather-app/env/wflow_[machine].env'. -# When generating a workflow experiment or running a workflow, users can use this file for a specific machine. +# For users' convenience, the python environment for the workflow can be activated by loading wflow_[PLATFORM] modulefile # For example, on Hera: -cd ufs-srweather-app/env -source wflow_hera.env +module load wflow_hera + +# Due to older version of Lmod, inconsistency with TCL modulefiles etc, you may have to activate +# conda manually using instructions that the previous module command prints. +# Hera is one of those systems, so execute: + +conda activate regional_workflow + +# After that we can setup an experiment in the directory + +cd regional_workflow/ush + +# Once we prepare experiment file config.sh, we can generate workflow using -cd ../regional_workflow/ush ./generate_FV3LAM_wflow.sh From 376b47aee0510965199274feb50c17552ac03b46 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 30 Apr 2022 13:17:49 +0000 Subject: [PATCH 22/26] Move cmake init out of lmod-setup.sh on odin --- lmod-setup.sh | 4 ---- modulefiles/build_odin_intel | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lmod-setup.sh b/lmod-setup.sh index bc3bdb2c06..deebb748e6 100644 --- a/lmod-setup.sh +++ b/lmod-setup.sh @@ -19,10 +19,6 @@ elif [ "$L_MACHINE" = odin ]; then module --initial_load --no_redirect restore #module use <$HOME>/ export MODULEPATH=/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles - - #module purge - export CMAKE=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake - export PATH=/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin:${PATH} else module purge fi diff --git a/modulefiles/build_odin_intel b/modulefiles/build_odin_intel index 06899da47e..25120a2af4 100644 --- a/modulefiles/build_odin_intel +++ b/modulefiles/build_odin_intel @@ -7,6 +7,9 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on odin" +prepend-path PATH "/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin" +setenv CMAKE "/home/yunheng.wang/tools/cmake-3.23.0-rc2/bin/cmake" + module load hpc/1.2.0 module load hpc-intel module load hpc-cray-mpich From bb06ae8969ecdcffcf56df10ad25a1693a4ebceb Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 30 Apr 2022 15:27:44 +0000 Subject: [PATCH 23/26] Also update markup language build documentation. --- docs/UsersGuide/source/BuildRunSRW.rst | 48 ++++++++++++++++---------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/docs/UsersGuide/source/BuildRunSRW.rst b/docs/UsersGuide/source/BuildRunSRW.rst index 5eee295f30..0adb42fdfe 100644 --- a/docs/UsersGuide/source/BuildRunSRW.rst +++ b/docs/UsersGuide/source/BuildRunSRW.rst @@ -96,7 +96,7 @@ The cloned repository contains the configuration files and sub-directories shown +--------------------------------+--------------------------------------------------------+ | ufs_srweather_app.settings.in | SRW App configuration summary | +--------------------------------+--------------------------------------------------------+ - | env | Contains build and workflow environment files | + | modulefiles | Contains build and workflow environment files | +--------------------------------+--------------------------------------------------------+ | docs | Contains release notes, documentation, and User's Guide| +--------------------------------+--------------------------------------------------------+ @@ -123,38 +123,48 @@ Run the executable that pulls in SRW App components from external repositories: -.. _SetUpBuild: +Build with ``devbuild.sh`` +========================== -Set up the Build Environment -============================ +On Level-1 systems, for which a modulefile is provided under ``modulefiles`` directory, we can build SRW App binaries with: + +.. code-block:: console -Before building the SRW App, the build environment must be set up for the user's specific platform. There is a set of common modules required to build the SRW App. These are located in the ``env/srw_common`` file. To load the set of common modules, run: + ./devbuild.sh --platform=hera + +If compiler auto-detection fails for some reason, specify it using .. code-block:: console - module use + ./devbuild.sh --platform=hera --compiler=intel -where ```` is the full path to the ``env`` directory. +If this method doesn't work, we will have to manually setup the environment, and build SRW app binaries with CMake. -Then, users must set up the platform-specific elements of the build environment. For Level 1 systems, scripts for loading the proper modules and/or setting the correct environment variables can be found in the ``env`` directory of the SRW App in files named ``build__.env``. Here is a sample directory listing of these build files: +.. _SetUpBuild: + +Set up the Build/Run Environment +================================ + +We need to setup our environment to run a workflow or to build the SRW app with CMake. Note that ``devbuild.sh`` does not prepare environment for workflow runs so this step is necessary even though binaries are built properly using ``devbuild.sh``. + +The build environment must be set up for the user's specific platform. First, we need to make sure ``Lmod`` is the app used for loading modulefiles. That is often the case on most systems, however, on some systems such as Gaea/Odin, the default modulefile loader is from Cray and we need to swap it for ``Lmod``. For example on Gaea, assuming a ``bash`` login shell, run: .. code-block:: console - $ ls -l env/ - -rw-rw-r-- 1 user ral 1228 Oct 9 10:09 build_cheyenne_intel.env - -rw-rw-r-- 1 user ral 1134 Oct 9 10:09 build_hera_intel.env - -rw-rw-r-- 1 user ral 1228 Oct 9 10:09 build_jet_intel.env - ... + source ./lmod-setup.sh gaea -On Level 1 systems, the commands in the ``build__.env`` files can be directly copy-pasted into the command line, or the file can be sourced from the ``ufs-srweather-app/env`` directory. For example, on Hera, run: +If you execute the above command on systems that don't need it, it will simply do a ``module purge``. From here on, we can assume, ``Lmod`` is ready to load modulefiles needed by the SRW app. -.. code-block:: +The modulefiles needed for building and running SRW App are located in ``modulefiles`` directory. To load the necessary modulefile for a specific ```` using ```` , run: + +.. code-block:: console - source env/build_hera_intel.env + module use + module load build__ -from the main ``ufs-srweather-app`` directory to source the appropriate file. +where ```` is the full path to the ``modulefiles`` directory. This will work on Level 1 systems, where a modulefile is available in the ``modulefiles`` directory. -On Level 2-4 systems, users will need to modify certain environment variables, such as the path to NCEP libraries, so that the SRW App can find and load the appropriate modules. For systems with Lmod installed, one of the current ``build__.env`` files can be copied and used as a template. To check whether Lmod is installed, run ``echo $LMOD_PKG``, and see if it outputs a path to the Lmod package. On systems without Lmod, users can modify or set the required environment variables with the ``export`` or ``setenv`` commands despending on whether they are using a bash or csh/tcsh shell, respectively: +On Level 2-4 systems, users will need to modify certain environment variables, such as the path to NCEP libraries, so that the SRW App can find and load the appropriate modules. For systems with Lmod installed, one of the current ``build__`` modulefiles can be copied and used as a template. To check whether Lmod is installed, run ``echo $LMOD_PKG``, and see if it outputs a path to the Lmod package. On systems without Lmod, users can modify or set the required environment variables with the ``export`` or ``setenv`` commands despending on whether they are using a bash or csh/tcsh shell, respectively: .. code-block:: @@ -599,7 +609,7 @@ The workflow requires Python 3 with the packages 'PyYAML', 'Jinja2', and 'f90nml .. code-block:: console - source ../../env/wflow_.env + module load wflow_ This command will activate the ``regional_workflow`` conda environment. The user should see ``(regional_workflow)`` in front of the Terminal prompt at this point. If this is not the case, activate the regional workflow from the ``ush`` directory by running: From 754cb6f5435a16dbca98c25069eabf4be761d583 Mon Sep 17 00:00:00 2001 From: "Daniel.Abdi" Date: Sat, 30 Apr 2022 22:39:28 -0400 Subject: [PATCH 24/26] Lmod setup script for both bash and tcsh login shells. --- devbuild.sh | 2 +- docs/INSTALL | 11 +++++---- docs/UsersGuide/source/BuildRunSRW.rst | 4 ++-- etc/lmod-setup.csh | 33 ++++++++++++++++++++++++++ etc/lmod-setup.sh | 32 +++++++++++++++++++++++++ lmod-setup.sh | 25 ------------------- 6 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 etc/lmod-setup.csh create mode 100644 etc/lmod-setup.sh delete mode 100644 lmod-setup.sh diff --git a/devbuild.sh b/devbuild.sh index 70c9aee5b4..12ee482d1c 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -243,7 +243,7 @@ if [ "${VERBOSE}" = true ]; then fi # Before we go on load modules, we first need to activate Lmod for some systems -source ${SRC_DIR}/lmod-setup.sh +source ${SRC_DIR}/etc/lmod-setup.sh # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" diff --git a/docs/INSTALL b/docs/INSTALL index 5b18d40845..4b659bb1d1 100644 --- a/docs/INSTALL +++ b/docs/INSTALL @@ -30,11 +30,14 @@ cd ufs-srweather-app/ # The above instructions will work atleast on Tier-1 systems, if not on all supported machines. # However, if it fails for some reason, we can build directly with cmake. -# First we need to make sure that there is a modulefile "build_[PLATFORM]_[COMPILER]" in the -# "modulefiles" directory. Also, on some systems (e.g. Gaea/Odin) that come with cray module system, -# we need to swap it for Lmod instead. Assuming the login shell is bash -source ./lmod-setup.sh PLATFORM +# First, we need to make sure that there is a modulefile "build_[PLATFORM]_[COMPILER]" in the +# "modulefiles" directory. Also, on some systems (e.g. Gaea/Odin) that come with cray module app, +# we may need to swap that for Lmod instead. Assuming your login shell is bash, run + +source etc/lmod-setup.sh PLATFORM + +# and if your login schell is csh/tcsh, source etc/lmod-setup.csh instead. # From here on, we can assume Lmod is loaded and ready to go. Then we load the specific # module for a given PLATFORM and COMPILER as follows diff --git a/docs/UsersGuide/source/BuildRunSRW.rst b/docs/UsersGuide/source/BuildRunSRW.rst index 0adb42fdfe..8f3f5e1b77 100644 --- a/docs/UsersGuide/source/BuildRunSRW.rst +++ b/docs/UsersGuide/source/BuildRunSRW.rst @@ -151,9 +151,9 @@ The build environment must be set up for the user's specific platform. First, we .. code-block:: console - source ./lmod-setup.sh gaea + source etc/lmod-setup.sh gaea -If you execute the above command on systems that don't need it, it will simply do a ``module purge``. From here on, we can assume, ``Lmod`` is ready to load modulefiles needed by the SRW app. +or if your login shell is ``csh`` or ``tcsh``, source ``etc/lmod-setup.csh`` instead. If you execute the above command on systems that don't need it, it will simply do a ``module purge``. From here on, we can assume, ``Lmod`` is ready to load modulefiles needed by the SRW app. The modulefiles needed for building and running SRW App are located in ``modulefiles`` directory. To load the necessary modulefile for a specific ```` using ```` , run: diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh new file mode 100644 index 0000000000..0b5c4581f8 --- /dev/null +++ b/etc/lmod-setup.csh @@ -0,0 +1,33 @@ +#!/bin/csh + +if ( $# == 0 ) then + set L_MACHINE=${MACHINE} +else + set L_MACHINE=$1 +endif + +echo "L_MACHINE:" $L_MACHINE + +if ( "$L_MACHINE" == macos ) then + # Choose lmod install location + setenv BASH_ENV "/opt/homebrew/opt/lmod/init/csh" + # setenv BASH_ENV "/usr/local/opt/lmod/init/csh" + source $BASH_ENV + +else if ( "$L_MACHINE" == gaea ) then + source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh" + +else if ( "$L_MACHINE" == odin ) then + module unload modules + unset -f module + + setenv BASH_ENV "/usr/local/lmod/8.3.1/init/csh" + source $BASH_ENV + setenv LMOD_SYSTEM_DEFAULT_MODULES "PrgEnv-intel:cray-mpich:intel:craype" + module --initial_load --no_redirect restore + setenv MODULEPATH "/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" + +else + module purge +endif + diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh new file mode 100644 index 0000000000..ed87edfaa3 --- /dev/null +++ b/etc/lmod-setup.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +if [ $# = 0 ]; then + L_MACHINE=${MACHINE} +else + L_MACHINE=$1 +fi + +if [ "$L_MACHINE" = macos ]; then + # Choose lmod install location + export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" + # export BASH_ENV="/usr/local/opt/lmod/init/bash" + source $BASH_ENV + +elif [ "$L_MACHINE" = gaea ]; then + source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" + +elif [ "$L_MACHINE" = odin ]; then + module unload modules + unset -f module + + export BASH_ENV=/usr/local/lmod/8.3.1/init/bash + source $BASH_ENV + export LMOD_SYSTEM_DEFAULT_MODULES="PrgEnv-intel:cray-mpich:intel:craype" + module --initial_load --no_redirect restore + #module use <$HOME>/ + export MODULEPATH="/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" + +else + module purge +fi + diff --git a/lmod-setup.sh b/lmod-setup.sh deleted file mode 100644 index deebb748e6..0000000000 --- a/lmod-setup.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -L_MACHINE=${MACHINE:-$1} - -if [ "$L_MACHINE" = macos ]; then - # Choose lmod install location - export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" - # export BASH_ENV="/usr/local/opt/lmod/init/bash" - source $BASH_ENV -elif [ "$L_MACHINE" = gaea ]; then - source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" -elif [ "$L_MACHINE" = odin ]; then - module unload modules - unset -f module - - export BASH_ENV=/usr/local/lmod/8.3.1/init/bash - source $BASH_ENV - export LMOD_SYSTEM_DEFAULT_MODULES=PrgEnv-intel:cray-mpich:intel:craype - module --initial_load --no_redirect restore - #module use <$HOME>/ - export MODULEPATH=/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles -else - module purge -fi - From 7eb75faa821796aab8ebbf2620633c829b619a7f Mon Sep 17 00:00:00 2001 From: "Daniel.Abdi" Date: Sun, 1 May 2022 05:11:31 -0400 Subject: [PATCH 25/26] Some fixes for tcsh login shell. --- docs/UsersGuide/source/BuildRunSRW.rst | 4 +++- etc/lmod-setup.csh | 17 ++++++++--------- etc/lmod-setup.sh | 8 ++++---- modulefiles/wflow_odin | 17 +++++++++++------ 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/docs/UsersGuide/source/BuildRunSRW.rst b/docs/UsersGuide/source/BuildRunSRW.rst index 8f3f5e1b77..5ac57920f9 100644 --- a/docs/UsersGuide/source/BuildRunSRW.rst +++ b/docs/UsersGuide/source/BuildRunSRW.rst @@ -96,7 +96,9 @@ The cloned repository contains the configuration files and sub-directories shown +--------------------------------+--------------------------------------------------------+ | ufs_srweather_app.settings.in | SRW App configuration summary | +--------------------------------+--------------------------------------------------------+ - | modulefiles | Contains build and workflow environment files | + | modulefiles | Contains build and workflow module files | + +--------------------------------+--------------------------------------------------------+ + | etc | Contains Lmod startup scripts | +--------------------------------+--------------------------------------------------------+ | docs | Contains release notes, documentation, and User's Guide| +--------------------------------+--------------------------------------------------------+ diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 0b5c4581f8..64935965bd 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -6,23 +6,22 @@ else set L_MACHINE=$1 endif -echo "L_MACHINE:" $L_MACHINE - if ( "$L_MACHINE" == macos ) then - # Choose lmod install location - setenv BASH_ENV "/opt/homebrew/opt/lmod/init/csh" - # setenv BASH_ENV "/usr/local/opt/lmod/init/csh" - source $BASH_ENV + set ENV="/opt/homebrew/opt/lmod/init/csh" + # setenv ENV "/usr/local/opt/lmod/init/csh" + source $ENV else if ( "$L_MACHINE" == gaea ) then - source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh" + set ENV="/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh" + source $ENV else if ( "$L_MACHINE" == odin ) then module unload modules unset -f module - setenv BASH_ENV "/usr/local/lmod/8.3.1/init/csh" - source $BASH_ENV + set ENV="/usr/local/lmod/8.3.1/init/csh" + source $ENV + setenv LMOD_SYSTEM_DEFAULT_MODULES "PrgEnv-intel:cray-mpich:intel:craype" module --initial_load --no_redirect restore setenv MODULEPATH "/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index ed87edfaa3..ce76c1f1bc 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -7,23 +7,23 @@ else fi if [ "$L_MACHINE" = macos ]; then - # Choose lmod install location export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" # export BASH_ENV="/usr/local/opt/lmod/init/bash" source $BASH_ENV elif [ "$L_MACHINE" = gaea ]; then - source "/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" + export BASH_ENV="/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" + source $BASH_ENV elif [ "$L_MACHINE" = odin ]; then module unload modules unset -f module - export BASH_ENV=/usr/local/lmod/8.3.1/init/bash + export BASH_ENV="/usr/local/lmod/8.3.1/init/bash" source $BASH_ENV + export LMOD_SYSTEM_DEFAULT_MODULES="PrgEnv-intel:cray-mpich:intel:craype" module --initial_load --no_redirect restore - #module use <$HOME>/ export MODULEPATH="/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" else diff --git a/modulefiles/wflow_odin b/modulefiles/wflow_odin index 6c84077fe2..ca240f84e1 100644 --- a/modulefiles/wflow_odin +++ b/modulefiles/wflow_odin @@ -10,13 +10,18 @@ module-whatis "Loads libraries needed for running SRW on Odin" if { [module-info mode load] } { # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! - if {[ system "/scratch/software/Odin/python/anaconda2/bin/conda shell.bash hook 2> /dev/null" ]} { + set shell [module-info shelltype] + set conda_path "/scratch/software/Odin/python/anaconda2" + if {$shell == "csh"} { + set conda_file "$conda_path/conda.csh" } else { - if {[ file exists "/scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh" ]} { - system "source /scratch/software/Odin/python/anaconda2/etc/profile.d/conda.sh;" - } else { - prepend-path PATH "/scratch/software/Odin/python/anaconda2/bin" - } + set conda_file "$conda_path/conda.sh" + } + + if {[ file exists "$conda_file" ]} { + system "source $conda_file;" + } else { + prepend-path PATH "$conda_path/bin" } # <<< conda initialize <<< From 20b19dbee40c6bb32b83d58603c7ac1d2a5e089e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 1 May 2022 16:37:38 +0000 Subject: [PATCH 26/26] Add singularity platform to lmod-setup --- etc/lmod-setup.csh | 8 ++++++++ etc/lmod-setup.sh | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 64935965bd..c25d94b165 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -11,6 +11,14 @@ if ( "$L_MACHINE" == macos ) then # setenv ENV "/usr/local/opt/lmod/init/csh" source $ENV + module purge + +else if ( "$L_MACHINE" == singularity ) then + set ENV="/usr/share/lmod/lmod/init/csh" + source $ENV + + module purge + else if ( "$L_MACHINE" == gaea ) then set ENV="/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh" source $ENV diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index ce76c1f1bc..fb8499c61c 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -11,6 +11,14 @@ if [ "$L_MACHINE" = macos ]; then # export BASH_ENV="/usr/local/opt/lmod/init/bash" source $BASH_ENV + module purge + +elif [ "$L_MACHINE" = singularity ]; then + export BASH_ENV="/usr/share/lmod/lmod/init/bash" + source $BASH_ENV + + module purge + elif [ "$L_MACHINE" = gaea ]; then export BASH_ENV="/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh" source $BASH_ENV