diff --git a/ci/spack.yaml b/ci/spack.yaml index deacdff0b5..647904108e 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -7,19 +7,19 @@ spack: - gcc@10:10 specs: - netcdf-c@4.9.2 - - netcdf-fortran@4.6.0 + - netcdf-fortran@4.6.1 - bufr@11.7.0 - bacio@2.4.1 - - w3emc@2.9.2 - - sp@2.3.3 - - ip@3.3.3 + - w3emc@2.10.0 + - sp@2.5.0 + - ip@4.3.0 - sigio@2.3.2 - sfcio@1.4.1 - nemsio@2.5.4 - wrf-io@1.2.0 - ncio@1.1.2 - - crtm@2.4.0 - - gsi-ncdiag@1.1.1 + - crtm@2.4.0.1 + - gsi-ncdiag@1.1.2 view: true concretizer: unify: true diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua deleted file mode 100644 index 1d903082a8..0000000000 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ /dev/null @@ -1,32 +0,0 @@ -help([[ -]]) - - -unload("ncarenv/1.3") -unload("intel/19.1.1") -unload("ncarcompilers/0.5.0") -unload("mpt/2.25") -unload("netcdf/4.8.1") - -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") - -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -local stack_gnu_ver=os.getenv("stack_gnu_ver") or "10.1.0" -local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.1" -local cmake_ver=os.getenv("cmake_ver") or "3.22.0" - -load(pathJoin("stack-gcc", stack_gnu_ver)) -load(pathJoin("stack-openmpi", stack_openmpi_ver)) -load(pathJoin("stack-python", stack_python_ver)) -load(pathJoin("cmake", cmake_ver)) -load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) -load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) - -load("gsi_common") - -pushenv("CFLAGS", "-xHOST") -pushenv("FFLAGS", "-xHOST") -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") - -whatis("Description: GSI environment on Cheyenne with GNU Compilers") diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua deleted file mode 100644 index 8c328e2b34..0000000000 --- a/modulefiles/gsi_cheyenne.intel.lua +++ /dev/null @@ -1,32 +0,0 @@ -help([[ -]]) - -unload("ncarenv/1.3") -unload("intel/19.1.1") -unload("ncarcompilers/0.5.0") -unload("mpt/2.25") -unload("netcdf/4.8.1") - -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") - -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -local stack_intel_ver=os.getenv("stack_intel_ver") or "19.1.1.217" -local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2019.7.217" -local cmake_ver=os.getenv("cmake_ver") or "3.22.0" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-mpi", stack_mpi_ver)) -load(pathJoin("stack-python", stack_python_ver)) -load(pathJoin("cmake", cmake_ver)) - -load("gsi_common") -load(pathJoin("prod-util", os.getenv("prod_util_ver") or "1.2.2")) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230911") - -pushenv("CFLAGS", "-xHOST") -pushenv("FFLAGS", "-xHOST") - -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") - -whatis("Description: GSI environment on Cheyenne with Intel Compilers") diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index d3365a98dc..cb49a43878 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -3,19 +3,19 @@ Load common modules to build GSI on all machines ]]) local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" +local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -local sp_ver=os.getenv("sp_ver") or "2.3.3" +local sp_ver=os.getenv("sp_ver") or "2.5.0" local ip_ver=os.getenv("ip_ver") or "4.3.0" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" -local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0.1" local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2" load(pathJoin("netcdf-c", netcdf_c_ver)) diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index 550b01ee7b..d85e79b005 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -1,16 +1,16 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") --Needed for openmpi build prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -local openblas_ver=os.getenv("openblas_ver") or "0.3.19" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" +local openblas_ver=os.getenv("openblas_ver") or "0.3.24" load(pathJoin("stack-gcc", stack_gnu_ver)) load(pathJoin("stack-openmpi", stack_openmpi_ver)) diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index abdc6e5623..b967c32f74 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_hercules.intel.lua b/modulefiles/gsi_hercules.intel.lua index bf29bc21db..c72323cbad 100644 --- a/modulefiles/gsi_hercules.intel.lua +++ b/modulefiles/gsi_hercules.intel.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_jet.intel.lua b/modulefiles/gsi_jet.intel.lua index 20b80ff61a..b2b98be7ff 100644 --- a/modulefiles/gsi_jet.intel.lua +++ b/modulefiles/gsi_jet.intel.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_orion.intel.lua b/modulefiles/gsi_orion.intel.lua index 80ec342c93..114e5f8ad1 100644 --- a/modulefiles/gsi_orion.intel.lua +++ b/modulefiles/gsi_orion.intel.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") -local stack_python_ver=os.getenv("python_ver") or "3.10.8" +local stack_python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_s4.intel.lua b/modulefiles/gsi_s4.intel.lua index a60ea3c16e..08e3d38516 100644 --- a/modulefiles/gsi_s4.intel.lua +++ b/modulefiles/gsi_s4.intel.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_wcoss2.intel.lua b/modulefiles/gsi_wcoss2.intel.lua index 8dde986e58..28b2f0d09d 100644 --- a/modulefiles/gsi_wcoss2.intel.lua +++ b/modulefiles/gsi_wcoss2.intel.lua @@ -20,7 +20,7 @@ local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" -local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0.1" local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) diff --git a/regression/regression_param.sh b/regression/regression_param.sh index 46d2647ac0..2ac615fc4a 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -36,11 +36,6 @@ case $machine in ;; Discover) sub_cmd="sub_discover" - ;; - Cheyenne) - sub_cmd="sub_cheyenne" - memnode=128 - numcore=36 ;; *) # EXIT out for unresolved machine echo "unknown $machine" @@ -73,9 +68,6 @@ case $regtest in elif [[ "$machine" = "Discover" ]]; then topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2" @@ -106,9 +98,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" @@ -139,9 +128,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" @@ -171,9 +157,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" @@ -204,9 +187,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" @@ -237,9 +217,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" @@ -270,9 +247,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" @@ -342,12 +316,6 @@ elif [[ "$machine" = "Gaea" ]]; then export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks" -elif [[ "$machine" = "Cheyenne" ]]; then - export OMP_STACKSIZE=1024M - export MPI_BUFS_PER_PROC=256 - export MPI_BUFS_PER_HOST=256 - export MPI_GROUP_MAX=256 - export APRUN="mpirun -v -np \$ntasks" elif [[ "$machine" = "wcoss2" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=2G diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 02ffb24b12..9d91b2c41f 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -30,9 +30,7 @@ else fi # Determine the machine -if [[ -d /glade ]]; then # Cheyenne - export machine="Cheyenne" -elif [[ -d /scratch1 ]]; then # Hera +if [[ -d /scratch1 ]]; then # Hera export machine="Hera" elif [[ -d /mnt/lfs4 || -d /jetmon || -d /mnt/lfs1 ]]; then # Jet export machine="Jet" @@ -66,20 +64,6 @@ case $machine in export check_resource="no" export accnt="nggps_emc" ;; - Cheyenne) - export queue="regular" - export noscrub="/glade/scratch/$LOGNAME/noscrub" - export group="global" - if [[ "$cmaketest" = "false" ]]; then - export basedir="/glade/scratch/$LOGNAME" - fi - export ptmp="/glade/scratch/$LOGNAME/$ptmpName" - - export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest" - - export check_resource="no" - export accnt="NRAL0032" - ;; wcoss2) export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}" if [ -d $local_or_default ]; then diff --git a/ush/module-setup.sh b/ush/module-setup.sh index d13da1efa3..c1893ab4ee 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -40,13 +40,6 @@ elif [[ $MACHINE_ID = wcoss2 ]]; then # We are on WCOSS2 module reset -elif [[ $MACHINE_ID = cheyenne* ]] ; then - # We are on NCAR Cheyenne - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh - fi - module purge - elif [[ $MACHINE_ID = stampede* ]] ; then # We are on TACC Stampede if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/sub_cheyenne b/ush/sub_cheyenne deleted file mode 100644 index 7389bfeb24..0000000000 --- a/ush/sub_cheyenne +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/sh --login -set -x -echo "starting sub_cheyenne" -usage="\ -Usage: $0 [options] executable [args] - where the options are: - -a account account (default: none) - -b binding run smt binding or not (default:NO) - -d dirin initial directory (default: cwd) - -e envars copy comma-separated environment variables - -g group group name - -i append standard input to command file - -j jobname specify jobname (default: executable basename) - -m machine machine on which to run (default: current) - -n write command file to stdout rather than submitting it - -o output specify output file (default: jobname.out) - -p procs[/nodes[/ppreq] - number of MPI tasks and optional nodes or Bblocking and - ppreq option (N or S) (defaults: serial, Bunlimited, S) - -q queue[/qpreq] queue name and optional requirement, e.g. dev/P - (defaults: 1 if serial or dev if parallel and none) - (queue 3 or 4 is dev or prod with twice tasks over ip) - (options: P=parallel, B=bigmem, b=batch) - -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) - -t timew wall time limit in [[hh:]mm:]ss format (default: 900) - -u userid userid to run under (default: self) - -v verbose mode - -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or - Thh[mm] (full, incremental, today or tomorrow) format - (default: now) -Function: This command submits a job to the batch queue." -subcmd="$*" -stdin=NO -nosub=NO -account="" -binding="NO" -dirin="" -envars="" -group="" -jobname="" -machine="" -output="" -procs=0 -nodes="" -ppreq="" -queue="" -qpreq="" -rmem="1024" -rcpu="1" -timew="900" -userid="" -verbose=NO -when="" -while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do - case $opt in - a) account="$OPTARG";; - b) binding="$OPTARG";; - d) dirin="$OPTARG";; - e) envars="$OPTARG";; - g) group="$OPTARG";; - i) stdin=YES;; - j) jobname=$OPTARG;; - m) machine="$OPTARG";; - n) nosub=YES;; - o) output=$OPTARG;; - p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; - q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; - r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; - t) timew=$OPTARG;; - u) userid=$OPTARG;; - v) verbose=YES;; - w) when=$OPTARG;; - \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; - esac -done -shift $(($OPTIND-1)) -if [[ $# -eq 0 ]];then - echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 -fi -exec=$1 -if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then - exec=$(which $exec) -fi -shift -args="$*" -bn=$(basename $exec) -export jobname=${jobname:-$bn} -output=${output:-$jobname.out} -myuser=$LOGNAME -myhost=$(hostname) - -DATA=/glade/scratch/$LOGNAME/tmp -mkdir -p $DATA - -timew=${timew:-01:20:00} -task_node=${task_node:-$procs} -size=$((nodes*task_node)) -envars=$envars -threads=${rcpu:-1} - -export TZ=GMT -cfile=$DATA/sub$$ -> $cfile -echo "#!/bin/sh --login" >> $cfile -echo "" >> $cfile -echo "#PBS -o $output" >> $cfile -echo "#PBS -N $jobname" >> $cfile -echo "#PBS -q $queue" >> $cfile -echo "#PBS -l walltime=$timew" >> $cfile -echo "#PBS -l select=$nodes:ncpus=$procs:mpiprocs=$procs" >> $cfile -echo "#PBS -j oe" >> $cfile -echo "#PBS -A $accnt" >> $cfile -echo "#PBS -V" >> $cfile - -echo "" >>$cfile -echo "export ntasks=$(( $nodes * $procs ))" >> $cfile -echo "export ppn=$procs" >> $cfile -echo "export threads=$threads" >> $cfile -echo "export OMP_NUM_THREADS=$threads" >> $cfile -echo "ulimit -s unlimited" >> $cfile -echo "" >>$cfile -echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile -echo "" >>$cfile - -echo "cfile = $cfile" -echo "source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh >> $cfile" -echo "module purge" >> $cfile -echo "module use $modulefiles" >> $cfile -echo "module load gsi_cheyenne.intel" >> $cfile -echo "module list" >> $cfile - -cat $exec >> $cfile - -if [[ $nosub = YES ]];then - cat $cfile - exit -elif [[ $verbose = YES ]];then - set -x - cat $cfile -fi - - -if [[ $stdin = YES ]];then - cat -fi >>$cfile -if [[ $nosub = YES ]];then - cat $cfile - exit -elif [[ $verbose = YES ]];then - set -x - cat $cfile -fi -qsub=${qsub:-qsub} - -ofile=$DATA/subout$$ ->$ofile -chmod 777 $ofile -$qsub $cfile >$ofile -rc=$? -cat $ofile -if [[ -w $SUBLOG ]];then - jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) - date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG -fi -rm $cfile $ofile -[[ $MKDATA = YES ]] && rmdir $DATA -echo "ending sub_cheyenne" -exit $rc - diff --git a/ush/sub_hercules b/ush/sub_hercules index e854d5727b..459b480559 100755 --- a/ush/sub_hercules +++ b/ush/sub_hercules @@ -119,7 +119,6 @@ echo "export ntasks=$(( $nodes * $procs ))" >> $cfile echo "export ppn=$procs" >> $cfile echo "export threads=$threads" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile -##echo "export OMP_STACKSIZE=2048M" >> $cfile echo "ulimit -s unlimited" >> $cfile echo "" >>$cfile @@ -130,7 +129,8 @@ echo ". /apps/other/lmod/lmod/init/sh" >> $cfile echo "module purge" >> $cfile echo "module use $modulefiles" >> $cfile echo "module load gsi_hercules.intel" >> $cfile -echo "module list" >> $cfile +#TODO reenable I_MPI_EXTRA_FILESYSTEM once regional ctests can properly handle parallel I/O on Hercules +echo "unset I_MPI_EXTRA_FILESYSTEM" >> $cfile echo "" >> $cfile cat $exec >> $cfile