From a261d766afa140d319ad801f7c99e776e6f08e54 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 14 Nov 2019 14:39:40 -0700 Subject: [PATCH 1/9] Start process of moving module loads out of J-jobs or ex-scripts and into into separate module files. Changes are listed below: 1) Add a new script (ush/load_modules.sh) that takes in the task name as an argument and uses the appropriate module file for that task (for the current machine) to load the necessary modules. 2) For the make_grid task and on hera only (so far), remove the module load commands from the ex-script (scripts/exregional_make_grid.sh) and move them to a new module file at $HOMErrfs/modulefiles/hera/make_grid. 3) In scripts/exregional_make_grid.sh, remove the module loads for theia. 4) In the rocoto XML template file (ush/templates/FV3SAR_wflow.xml), create a new ENTITY (LOAD_MODULES_SCR) that contains the name of the new script that loads the module files for a given task (ush/load_modules.sh). Also, create new ENTITYs that contain the names of the workflow tasks (this is just for convenience since the task name is going to appear in several places in the XML in each task). 5) For the make_grid task only (so far), I changed the "..." in "..." in the rocoto XML template to contain calls to two scripts -- first the load_modules.sh script with the task name as an argument, and then the J-job for the task -- instead of just one (the J-job for the task). This still needs to be testd. --- scripts/exregional_make_grid.sh | 62 ++++++------------ ush/load_modules.sh | 93 +++++++++++++++++++++++++++ ush/templates/FV3SAR_wflow.xml | 107 +++++++++++++++++++------------- 3 files changed, 176 insertions(+), 86 deletions(-) create mode 100644 ush/load_modules.sh diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 45f0e5d859..c743834546 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -132,50 +132,28 @@ case $MACHINE in ;; -"THEIA") +#"HERA") +## +# { save_shell_opts; set +x; } > /dev/null 2>&1 # - { save_shell_opts; set +x; } > /dev/null 2>&1 - - . /apps/lmod/lmod/init/sh - module purge - module load intel/16.1.150 - module load impi - module load hdf5/1.8.14 - module load netcdf/4.3.0 - module list - - { restore_shell_opts; } > /dev/null 2>&1 - - export APRUN="time" - export topo_dir="/scratch4/NCEPDEV/global/save/glopara/svn/fv3gfs/fix/fix_orog" - - ulimit -s unlimited - ulimit -a - ;; - - -"HERA") +# . /apps/lmod/lmod/init/sh +# module purge +# module load intel/18.0.5.274 +## module load netcdf/4.6.1 +## module load hdf5/1.10.4 +# module load netcdf/4.7.0 +# module load hdf5/1.10.5 +# module list +# +# { restore_shell_opts; } > /dev/null 2>&1 +# +# export APRUN="time" +# export topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog" +# +# ulimit -s unlimited +# ulimit -a +# ;; # - { save_shell_opts; set +x; } > /dev/null 2>&1 - - . /apps/lmod/lmod/init/sh - module purge - module load intel/18.0.5.274 -# module load netcdf/4.6.1 -# module load hdf5/1.10.4 - module load netcdf/4.7.0 - module load hdf5/1.10.5 - module list - - { restore_shell_opts; } > /dev/null 2>&1 - - export APRUN="time" - export topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog" - - ulimit -s unlimited - ulimit -a - ;; - "JET") # diff --git a/ush/load_modules.sh b/ush/load_modules.sh new file mode 100644 index 0000000000..85fa8a6c76 --- /dev/null +++ b/ush/load_modules.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +#set -x -u -e +#date +# +#----------------------------------------------------------------------- +# +# Source the variable definitions script and the function definitions +# file. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# +task_name="$1" + +#. ${HOMErrfs}/rocoto/machine-setup.sh +#export machine=${target} + +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# +case "$MACHINE" in +# + "WCOSS_C") + . /opt/modules/default/init/sh + ;; +# + "DELL") + . /usrx/local/prod/lmod/lmod/init/sh + ;; +# + "HERA") + . /apps/lmod/lmod/init/sh + ;; +# + "JET") + . /apps/lmod/lmod/init/sh + ;; +# + *) + print_err_msg_exit " +The script to source to initialize lmod (module loads) has not yet been +specified for the current machine (MACHINE): + MACHINE = \"$MACHINE\"" + ;; +# +esac +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# +module use ${HOMErrfs}/modulefiles/${MACHINE} +module load ${task_name} +module list + +#exec "$@" + diff --git a/ush/templates/FV3SAR_wflow.xml b/ush/templates/FV3SAR_wflow.xml index 6eedf4a686..cf77581f87 100644 --- a/ush/templates/FV3SAR_wflow.xml +++ b/ush/templates/FV3SAR_wflow.xml @@ -77,6 +77,24 @@ Variables that are not modified by the workflow generation script. &QUEUE_HPSS;&ACCOUNT;"> &QUEUE_FCST;&ACCOUNT;"> + + + + + + + + + + + + + + ]> @@ -91,16 +109,17 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_MAKE_GRID; &RSRV_DEFAULT; - - &JOBSDIR;/JREGIONAL_MAKE_GRID + { &USHDIR/&LOAD_MODULES_SCR; &TN_MAKE_GRID;; &JOBSDIR;/JREGIONAL_MAKE_GRID } &PROC_MAKE_GRID; - make_grid - &LOGDIR;/make_grid.log + &TN_MAKE_GRID; + &LOGDIR;/&TN_MAKE_GRID;.log + MACHINE&MACHINE; + HOMErrfs&HOMErrfs; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -113,15 +132,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_MAKE_OROG; &RSRV_DEFAULT; &JOBSDIR;/JREGIONAL_MAKE_OROG &PROC_MAKE_OROG; - make_orog - &LOGDIR;/make_orog.log + &TN_MAKE_OROG; + &LOGDIR;/&TN_MAKE_OROG;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -142,15 +161,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_MAKE_SFC_CLIMO; &RSRV_DEFAULT; &JOBSDIR;/JREGIONAL_MAKE_SFC_CLIMO &PROC_MAKE_SFC_CLIMO; - make_sfc_climo - &LOGDIR;/make_sfc_climo.log + &TN_MAKE_SFC_CLIMO; + &LOGDIR;/&TN_MAKE_SFC_CLIMO;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -164,8 +183,8 @@ Variables that are not modified by the workflow generation script. &RUN_TASK_MAKE_GRID;FALSE - - &LOGDIR;/make_orog_task_complete.txt + + &LOGDIR;/&TN_MAKE_OROG;_task_complete.txt &RUN_TASK_MAKE_OROG;FALSE @@ -176,15 +195,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_GET_EXTRN_MDL_FILES; &RSRV_HPSS; &JOBSDIR;/JREGIONAL_GET_EXTRN_FILES &PROC_GET_EXTRN_MDL_FILES; - get_files_ICS - &LOGDIR;/get_files_ICS_@Y@m@d@H.log + &TN_GET_EXTRN_ICS; + &LOGDIR;/&TN_GET_EXTRN_ICS;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; CYCLE_DIR&CYCLE_DIR; @@ -198,15 +217,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_GET_EXTRN_MDL_FILES; &RSRV_HPSS; &JOBSDIR;/JREGIONAL_GET_EXTRN_FILES &PROC_GET_EXTRN_MDL_FILES; - get_files_LBCS - &LOGDIR;/get_files_LBCS_@Y@m@d@H.log + &TN_GET_EXTRN_LBCS; + &LOGDIR;/&TN_GET_EXTRN_LBCS;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; CYCLE_DIR&CYCLE_DIR; @@ -220,15 +239,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_MAKE_ICS_SURF_LBC0; &RSRV_DEFAULT; &JOBSDIR;/JREGIONAL_MAKE_IC_LBC0 &PROC_MAKE_ICS_SURF_LBC0; - make_ICS_surf_LBC0 - &LOGDIR;/make_ICS_surf_LBC0_@Y@m@d@H.log + &TN_MAKE_ICS; + &LOGDIR;/&TN_MAKE_ICS;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; CYCLE_DIR&CYCLE_DIR; @@ -237,20 +256,20 @@ Variables that are not modified by the workflow generation script. - + &LOGDIR;/make_grid_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE - - &LOGDIR;/make_orog_task_complete.txt + + &LOGDIR;/&TN_MAKE_OROG;_task_complete.txt &RUN_TASK_MAKE_OROG;FALSE - - &LOGDIR;/make_sfc_climo_task_complete.txt + + &LOGDIR;/&TN_MAKE_SFC_CLIMO;_task_complete.txt &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -261,15 +280,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_MAKE_LBC1_TO_LBCN; &RSRV_DEFAULT; &JOBSDIR;/JREGIONAL_MAKE_LBC1_TO_LBCN &PROC_MAKE_LBC1_TO_LBCN; - make_LBC1_to_LBCN - &LOGDIR;/make_LBC1_to_LBCN_@Y@m@d@H.log + &TN_MAKE_LBCS; + &LOGDIR;/&TN_MAKE_LBCS;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; CYCLE_DIR&CYCLE_DIR; @@ -278,20 +297,20 @@ Variables that are not modified by the workflow generation script. - + &LOGDIR;/make_grid_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE - - &LOGDIR;/make_orog_task_complete.txt + + &LOGDIR;/&TN_MAKE_OROG;_task_complete.txt &RUN_TASK_MAKE_OROG;FALSE - - &LOGDIR;/make_sfc_climo_task_complete.txt + + &LOGDIR;/&TN_MAKE_SFC_CLIMO;_task_complete.txt &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -302,15 +321,15 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &RSRC_RUN_FV3; &RSRV_RUN_FV3; &JOBSDIR;/JREGIONAL_RUN_FV3 &PROC_RUN_FV3; - run_FV3 - &LOGDIR;/run_FV3_@Y@m@d@H.log + &TN_RUN_FCST; + &LOGDIR;/&TN_RUN_FCST;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; CYCLE_DIR&CYCLE_DIR; @@ -319,8 +338,8 @@ Variables that are not modified by the workflow generation script. - - + + @@ -329,19 +348,19 @@ Variables that are not modified by the workflow generation script. ************************************************************************ ************************************************************************ --> - + &FHR; - + &RSRC_POST; &RSRV_DEFAULT; &JOBSDIR;/JREGIONAL_RUN_POST &PROC_POST; - run_post_#fhr# - &LOGDIR;/run_post_#fhr#_@Y@m@d@H.log + &TN_RUN_POST;_#fhr# + &LOGDIR;/&TN_RUN_POST;_#fhr#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; CYCLE_DIR&CYCLE_DIR; From 24825c6094506f76bee9ae22a399f02ee7b3ba17 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 14 Nov 2019 15:35:17 -0700 Subject: [PATCH 2/9] Add module file missed in last commit. --- modulefiles/hera/make_grid | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 modulefiles/hera/make_grid diff --git a/modulefiles/hera/make_grid b/modulefiles/hera/make_grid new file mode 100644 index 0000000000..240cbb10e9 --- /dev/null +++ b/modulefiles/hera/make_grid @@ -0,0 +1,11 @@ +#%Module##################################################### +## Module file for make_grid task. +############################################################# + +. /apps/lmod/lmod/init/sh +module purge +module load intel/18.0.5.274 +module load netcdf/4.7.0 +module load hdf5/1.10.5 +module list + From cf78627478faa3d91eb34e91d9d954e365ce635c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 15 Nov 2019 14:58:36 -0700 Subject: [PATCH 3/9] Changes for the make_grid task to get it to work with the module loads moved out of the J-job (on hera only). Now works. Details below. 1) modulefiles/hera/make_grid: Remove the sourcing of the script that defines the module() shell function out of the module file for this task ($HOMErrfs/modulefiles/hera/make_grid). That is already done in the ush/load_module_run_task.sh script (see below). 2) scripts/exregional_make_grid.sh: Uncomment parts of the HERA stanza in the case statement. We need to do this because this stanza does more than just load modules (which is now done in the ush/load_module_run_task.sh script); it defines certain variables (e.g. APRUN, topo_dir). 3) ush/load_modules.sh: a) Rename to ush/load_modules_run_task.sh because it not only loads the modules but also runs the make_grid (or whatever specified) task. b) Add appropriate error checking, usage message, informational messages, and comments. 4) ush/templates/FV3SAR_wflow.xml: a) Fix the way the J-job for the make_grid task is called -- now use the ush/load_modules_run_task.sh script to load modules and run the ask. b) Remove unnecessary settings of the environment variables MACHINE and HOMErrfs (these are available to the ush/load_moudles_run_task.sh script via sourcing the global variable definitions file). --- modulefiles/hera/make_grid | 2 - scripts/exregional_make_grid.sh | 9 +- ush/load_modules.sh | 93 ------------------ ush/load_modules_run_task.sh | 162 ++++++++++++++++++++++++++++++++ ush/templates/FV3SAR_wflow.xml | 33 ++++++- 5 files changed, 195 insertions(+), 104 deletions(-) delete mode 100644 ush/load_modules.sh create mode 100755 ush/load_modules_run_task.sh diff --git a/modulefiles/hera/make_grid b/modulefiles/hera/make_grid index 240cbb10e9..66368b2aaa 100644 --- a/modulefiles/hera/make_grid +++ b/modulefiles/hera/make_grid @@ -2,10 +2,8 @@ ## Module file for make_grid task. ############################################################# -. /apps/lmod/lmod/init/sh module purge module load intel/18.0.5.274 module load netcdf/4.7.0 module load hdf5/1.10.5 -module list diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index c743834546..8f5b08d694 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -132,8 +132,8 @@ case $MACHINE in ;; -#"HERA") -## +"HERA") +# # { save_shell_opts; set +x; } > /dev/null 2>&1 # # . /apps/lmod/lmod/init/sh @@ -148,11 +148,12 @@ case $MACHINE in # { restore_shell_opts; } > /dev/null 2>&1 # # export APRUN="time" -# export topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog" + APRUN="time" + topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog" # # ulimit -s unlimited # ulimit -a -# ;; + ;; # "JET") diff --git a/ush/load_modules.sh b/ush/load_modules.sh deleted file mode 100644 index 85fa8a6c76..0000000000 --- a/ush/load_modules.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -#set -x -u -e -#date -# -#----------------------------------------------------------------------- -# -# Source the variable definitions script and the function definitions -# file. -# -#----------------------------------------------------------------------- -# -. ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# -task_name="$1" - -#. ${HOMErrfs}/rocoto/machine-setup.sh -#export machine=${target} - -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# -case "$MACHINE" in -# - "WCOSS_C") - . /opt/modules/default/init/sh - ;; -# - "DELL") - . /usrx/local/prod/lmod/lmod/init/sh - ;; -# - "HERA") - . /apps/lmod/lmod/init/sh - ;; -# - "JET") - . /apps/lmod/lmod/init/sh - ;; -# - *) - print_err_msg_exit " -The script to source to initialize lmod (module loads) has not yet been -specified for the current machine (MACHINE): - MACHINE = \"$MACHINE\"" - ;; -# -esac -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# -module use ${HOMErrfs}/modulefiles/${MACHINE} -module load ${task_name} -module list - -#exec "$@" - diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh new file mode 100755 index 0000000000..d8c8434639 --- /dev/null +++ b/ush/load_modules_run_task.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions script and the function definitions +# file. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Check arguments. +# +#----------------------------------------------------------------------- +# +if [ "$#" -ne 2 ]; then + + print_err_msg_exit " +Incorrect number of arguments specified: + + Number of arguments specified: $# + +Usage: + + ${scrfunc_fn} task_name jjob_fp + +where the arguments are defined as follows: + + task_name: + The name of the rocoto task for which this script will load modules + and launch the J-job. + + jjob_fp + The full path to the J-job script corresponding to task_name. This + script will launch this J-job using the \"exec\" command (which will + first terminate this script and then launch the j-job; see man page of + the \"exec\" command). +" + +fi +# +#----------------------------------------------------------------------- +# +# Call the script that defines the module() function. This is needed so +# we can perform "module load ..." calls later below. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" " +Initializing the shell function \"module()\" (and others) in order to be +able to use \"module load ...\" to load necessary modules ..." + +case "$MACHINE" in +# + "WCOSS_C") + . /opt/modules/default/init/sh + ;; +# + "DELL") + . /usrx/local/prod/lmod/lmod/init/sh + ;; +# + "HERA") + . /apps/lmod/lmod/init/sh + ;; +# + "JET") + . /apps/lmod/lmod/init/sh + ;; +# + *) + print_err_msg_exit " +The script to source to initialize lmod (module loads) has not yet been +specified for the current machine (MACHINE): + MACHINE = \"$MACHINE\"" + ;; +# +esac +# +#----------------------------------------------------------------------- +# +# Get the task name. Then shift the argument list so that the firs ar- +# gument (the task name) gets dropped from the start of the arguments +# list. This results in the shell variable $@ containing only the re- +# maining arguments (which in this case should consist of the full path +# to the J-job to call). +# +#----------------------------------------------------------------------- +# +task_name="$1" +jjob_fp="$2" +# +#----------------------------------------------------------------------- +# +# Purge modules and load the module file for the specified task on the +# current machine. Note that the path to this module file is given by +# +# $HOMErrfs/modulefiles/$machine/${task_name} +# +# where HOMErrfs is the workflow home directory, machine is the name of +# the current machine in lowercase, and task_name is the name of the +# task that this script will launch (via the exec command below). +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" " +Loading modules for task \"${task_name}\" ..." + +machine=${MACHINE,,} +module purge +module use $HOMErrfs/modulefiles/$machine +module load ${task_name} +module list +# +#----------------------------------------------------------------------- +# +# Use the exec ocmmand to terminate the current script and launch the +# J-job for the specified task. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" " +Launching J-job (jjob_fp) for task \"${task_name}\" ... + jjob_fp = \"${jjob_fp}\" +" +exec "${jjob_fp}" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + + diff --git a/ush/templates/FV3SAR_wflow.xml b/ush/templates/FV3SAR_wflow.xml index cf77581f87..316af64733 100644 --- a/ush/templates/FV3SAR_wflow.xml +++ b/ush/templates/FV3SAR_wflow.xml @@ -78,9 +78,10 @@ Variables that are not modified by the workflow generation script. &QUEUE_FCST;&ACCOUNT;"> - + + + + &LOAD_MODULES_RUN_TASK_FP; "&TN_MAKE_GRID;" "&JOBSDIR;/JREGIONAL_MAKE_GRID" &PROC_MAKE_GRID; &TN_MAKE_GRID; &LOGDIR;/&TN_MAKE_GRID;.log - MACHINE&MACHINE; - HOMErrfs&HOMErrfs; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d From c5bf612a082c76ff52db00ff3bce75b70078f96a Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 15 Nov 2019 15:03:57 -0700 Subject: [PATCH 4/9] Improvements to tests/run_experiments.sh: (1) Remvoe variable TESTSDIR since it is not needed. (2) Introduce variable baseline_configs_dir. This is the directory in which the experiment/workflow configuration files for the baselines are assumed to be located. (3) Add argument check, usage message, and file existence check. (4) Clean up informational messages. --- tests/run_experiments.sh | 135 ++++++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 52 deletions(-) diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index f09fb6eac2..e90cfae87c 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -16,13 +16,13 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) #----------------------------------------------------------------------- # # The current script should be located in the "tests" subdirectory of -# the workflow directory, which we denote by HOMErrfs. Thus, the work- -# flow directory (HOMErrfs) is the one above the directory of the cur- +# the workflow directory, which we denote by homerrfs. Thus, the work- +# flow directory (homerrfs) is the one above the directory of the cur- # rent script. Set HOMRErrfs accordingly. # #----------------------------------------------------------------------- # -HOMErrfs=${scrfunc_dir%/*} +homerrfs=${scrfunc_dir%/*} # #----------------------------------------------------------------------- # @@ -30,8 +30,8 @@ HOMErrfs=${scrfunc_dir%/*} # #----------------------------------------------------------------------- # -USHDIR="$HOMErrfs/ush" -TESTSDIR="$HOMErrfs/tests" +ushdir="$homerrfs/ush" +baseline_configs_dir="$homerrfs/tests/baseline_configs" # #----------------------------------------------------------------------- # @@ -39,7 +39,7 @@ TESTSDIR="$HOMErrfs/tests" # #----------------------------------------------------------------------- # -. $USHDIR/source_util_funcs.sh +. $ushdir/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -56,12 +56,54 @@ TESTSDIR="$HOMErrfs/tests" # #----------------------------------------------------------------------- # -MACHINE="HERA" -ACCOUNT="gsd-fv3" +machine="HERA" +account="gsd-fv3" -USE_CRON_TO_RELAUNCH="TRUE" -#USE_CRON_TO_RELAUNCH="FALSE" -CRON_RELAUNCH_INTVL_MNTS="02" +use_cron_to_relaunch="TRUE" +#use_cron_to_relaunch="FALSE" +cron_relaunch_intvl_mnts="02" +# +#----------------------------------------------------------------------- +# +# Check arguments. +# +#----------------------------------------------------------------------- +# + if [ "$#" -ne 1 ]; then + + print_err_msg_exit " +Incorrect number of arguments specified: + + Number of arguments specified: $# + +Usage: + + ${scrfunc_fn} expts_file + +where expts_file is the name of the file containing the list of experi- +ments to run. If expts_file is the absolute path to a file, it is used +as is. If it is a relative path (including just a file name), it is as- +sumed to be given relative to the path from which this script is called. +" + + fi +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# + expts_file="$1" + expts_list_fp=$( readlink -f "${expts_file}" ) + + if [ ! -f "${expts_list_fp}" ]; then + print_err_msg_exit "\ +The experiments list file (expts_file) specified as an argument to this +script (and with full path given by expts_list_fp) does not exist: + expts_file = \"${expts_file}\" + expts_list_fp = \"${expts_list_fp}\"" + fi # #----------------------------------------------------------------------- # @@ -86,24 +128,20 @@ CRON_RELAUNCH_INTVL_MNTS="02" # #----------------------------------------------------------------------- # -EXPTS_LIST_FN="expts_list.txt" -EXPTS_LIST_FP="${TESTSDIR}/${EXPTS_LIST_FN}" - print_info_msg " Reading in list of forecast experiments from file - EXPTS_LIST_FP = \"${EXPTS_LIST_FP}\" + expts_list_fp = \"${expts_list_fp}\" and storing result in the array \"all_lines\" (one array element per expe- riment)..." -readarray -t all_lines < "${EXPTS_LIST_FP}" +readarray -t all_lines < "${expts_list_fp}" all_lines_str=$( printf "\'%s\'\n" "${all_lines[@]}" ) print_info_msg " -All lines from experiments list file EXPTS_LIST_FP read in, where: - - EXPTS_LIST_FP = \"${EXPTS_LIST_FP}\" - -Contents of file (line by line, before any processing) are: +All lines from experiments list file (expts_list_fp) read in, where: + expts_list_fp = \"${expts_list_fp}\" +Contents of file are (line by line, each line within single quotes, and +before any processing): ${all_lines_str} " @@ -119,7 +157,7 @@ ${all_lines_str} #----------------------------------------------------------------------- # expts_list=() -field_separator="\|" # Need backslash as an escape sequence in the sed commands. +field_separator="\|" # Need backslash as an escape sequence in the sed commands below. j=0 num_lines="${#all_lines[@]}" @@ -165,12 +203,9 @@ num_expts="${#expts_list[@]}" expts_list_str=$( printf " \'%s\'\n" "${expts_list[@]}" ) print_info_msg " After processing, the number of experiments to run (num_expts) is: - num_expts = ${num_expts} - The list of forecast experiments to run (one experiment per line) is gi- ven by: - ${expts_list_str} " # @@ -185,14 +220,7 @@ ${expts_list_str} for (( i=0; i<=$((num_expts-1)); i++ )); do print_info_msg " - -Processing experiment #$((${i}+1)): ------------------------- - -The experiment specification line for this experiment is given by: - - ${expts_list[$i]} -" +Processing experiment \"${expts_list[$i]}\" ..." # # Get the name of the baseline on which the current experiment is based. # Then save the remainder of the current element of expts_list in the @@ -245,7 +273,7 @@ The experiment specification line for this experiment is given by: # This will be modified to obtain the configuration file for the current # experiment. # - baseline_config_fp="${TESTSDIR}/baseline_configs/config.${baseline_name}.sh" + baseline_config_fp="${baseline_configs_dir}/config.${baseline_name}.sh" # # Print out an error message and exit if a configuration file for the # current baseline does not exist. @@ -255,7 +283,8 @@ The experiment specification line for this experiment is given by: The experiment/workflow configuration file (baseline_config_fp) for the specified baseline (baseline_name) does not exist: baseline_name = \"${baseline_name}\" - baseline_config_fp = \"${baseline_config_fp}\"" + baseline_config_fp = \"${baseline_config_fp}\" +Please correct and rerun." fi # # We require that EXPT_SUBDIR in the configuration file for the baseline @@ -289,37 +318,37 @@ the name of the baseline (baseline_name): fi done # -# Reset EXPT_SUBDIR to the name of the current experiment. Below, we -# will write this to the configuration file for the current experiment. +# Set expt_subdir to the name of the current experiment. Below, we will +# write this to the configuration file for the current experiment. # - EXPT_SUBDIR="${expt_name}" + expt_subdir="${expt_name}" # # Create a configuration file for the current experiment. We do this by # first copying the baseline configuration file and then modifying the # the values of those variables within it that are different between the # baseline and the experiment. # - expt_config_fp="${USHDIR}/config.${expt_name}.sh" + expt_config_fp="$ushdir/config.${expt_name}.sh" cp_vrfy "${baseline_config_fp}" "${expt_config_fp}" - set_bash_param "${expt_config_fp}" "MACHINE" "$MACHINE" - set_bash_param "${expt_config_fp}" "ACCOUNT" "$ACCOUNT" - set_bash_param "${expt_config_fp}" "USE_CRON_TO_RELAUNCH" "${USE_CRON_TO_RELAUNCH}" - set_bash_param "${expt_config_fp}" "CRON_RELAUNCH_INTVL_MNTS" "${CRON_RELAUNCH_INTVL_MNTS}" - set_bash_param "${expt_config_fp}" "EXPT_SUBDIR" "${EXPT_SUBDIR}" + set_bash_param "${expt_config_fp}" "MACHINE" "$machine" + set_bash_param "${expt_config_fp}" "ACCOUNT" "$account" + set_bash_param "${expt_config_fp}" "USE_CRON_TO_RELAUNCH" "${use_cron_to_relaunch}" + set_bash_param "${expt_config_fp}" "CRON_RELAUNCH_INTVL_MNTS" "${cron_relaunch_intvl_mnts}" + set_bash_param "${expt_config_fp}" "EXPT_SUBDIR" "${expt_subdir}" printf "" for (( j=0; j<${num_mod_vars}; j++ )); do set_bash_param "${expt_config_fp}" "${modvar_name[$j]}" "${modvar_value[$j]}" done # -# Create a symlink called "config.sh" in USHDIR that points to the cur- +# Create a symlink called "config.sh" in ushdir that points to the cur- # rent experiment's configuration file. This must be done because the # experiment/workflow generation script assumes that this is the name # and location of the configuration file to use to generate a new expe- # riment and corresponding workflow. # - ln_vrfy -fs "${expt_config_fp}" "$USHDIR/config.sh" + ln_vrfy -fs "${expt_config_fp}" "$ushdir/config.sh" # #----------------------------------------------------------------------- # @@ -328,12 +357,14 @@ the name of the baseline (baseline_name): # #----------------------------------------------------------------------- # + log_fp="$ushdir/log.generate_FV3SAR_wflow.${expt_name}" print_info_msg " Generating experiment with name: - expt_name = \"${expt_name}\"" + expt_name = \"${expt_name}\" +Log file for generation step is: + log_fp = \"${log_fp}\"" - log_fp="$USHDIR/log.generate_FV3SAR_wflow.${expt_name}" - $USHDIR/generate_FV3SAR_wflow.sh 2>&1 >& "${log_fp}" || { \ + $ushdir/generate_FV3SAR_wflow.sh > "${log_fp}" 2>&1 || { \ print_err_msg_exit "\ Could not generate an experiment/workflow for the test specified by expt_name: @@ -352,9 +383,9 @@ log_fp: # #----------------------------------------------------------------------- # - EXPTDIR=$( readlink -f "$HOMErrfs/../expt_dirs/${EXPT_SUBDIR}" ) - mv_vrfy "${expt_config_fp}" "${EXPTDIR}" - mv_vrfy "${log_fp}" "${EXPTDIR}" + exptdir=$( readlink -f "$homerrfs/../expt_dirs/${expt_subdir}" ) + mv_vrfy "${expt_config_fp}" "${exptdir}" + mv_vrfy "${log_fp}" "${exptdir}" done # From 884dddf7fd621463c713bf40154dfcc8cc333175 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 18 Nov 2019 11:59:32 -0700 Subject: [PATCH 5/9] Rename tasks and corresponding J-jobs and ex-scripts to be shorter and more similar to EMC's workflow. --- ...EGIONAL_MAKE_IC_LBC0 => JREGIONAL_MAKE_ICS} | 2 +- ...L_MAKE_LBC1_TO_LBCN => JREGIONAL_MAKE_LBCS} | 2 +- jobs/{JREGIONAL_RUN_FV3 => JREGIONAL_RUN_FCST} | 2 +- ..._make_ic_lbc0.sh => exregional_make_ics.sh} | 0 ...lbc1_to_lbcn.sh => exregional_make_lbcs.sh} | 0 ...ional_run_fv3.sh => exregional_run_fcst.sh} | 0 ush/generate_FV3SAR_wflow.sh | 4 ++-- ush/templates/FV3SAR_wflow.xml | 18 +++++++++--------- 8 files changed, 14 insertions(+), 14 deletions(-) rename jobs/{JREGIONAL_MAKE_IC_LBC0 => JREGIONAL_MAKE_ICS} (99%) rename jobs/{JREGIONAL_MAKE_LBC1_TO_LBCN => JREGIONAL_MAKE_LBCS} (99%) rename jobs/{JREGIONAL_RUN_FV3 => JREGIONAL_RUN_FCST} (98%) rename scripts/{exregional_make_ic_lbc0.sh => exregional_make_ics.sh} (100%) rename scripts/{exregional_make_lbc1_to_lbcn.sh => exregional_make_lbcs.sh} (100%) rename scripts/{exregional_run_fv3.sh => exregional_run_fcst.sh} (100%) diff --git a/jobs/JREGIONAL_MAKE_IC_LBC0 b/jobs/JREGIONAL_MAKE_ICS similarity index 99% rename from jobs/JREGIONAL_MAKE_IC_LBC0 rename to jobs/JREGIONAL_MAKE_ICS index 465b7cf49f..eeaa95f43d 100755 --- a/jobs/JREGIONAL_MAKE_IC_LBC0 +++ b/jobs/JREGIONAL_MAKE_ICS @@ -216,7 +216,7 @@ EXTRN_MDL_FILES_DIR="${CYCLE_DIR}/${EXTRN_MDL_NAME_ICS}/ICS" # EXTRN_MDL_FNS_str="( "$( printf "\"%s\" " "${EXTRN_MDL_FNS[@]}" )")" -$SCRIPTSDIR/exregional_make_ic_lbc0.sh \ +$SCRIPTSDIR/exregional_make_ics.sh \ EXTRN_MDL_FNS="${EXTRN_MDL_FNS_str}" \ EXTRN_MDL_FILES_DIR="${EXTRN_MDL_FILES_DIR}" \ EXTRN_MDL_CDATE="${EXTRN_MDL_CDATE}" \ diff --git a/jobs/JREGIONAL_MAKE_LBC1_TO_LBCN b/jobs/JREGIONAL_MAKE_LBCS similarity index 99% rename from jobs/JREGIONAL_MAKE_LBC1_TO_LBCN rename to jobs/JREGIONAL_MAKE_LBCS index 3348626a3e..e29919d6fe 100755 --- a/jobs/JREGIONAL_MAKE_LBC1_TO_LBCN +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -218,7 +218,7 @@ EXTRN_MDL_FILES_DIR="${CYCLE_DIR}/${EXTRN_MDL_NAME_LBCS}/LBCS" EXTRN_MDL_FNS_str="( "$( printf "\"%s\" " "${EXTRN_MDL_FNS[@]}" )")" EXTRN_MDL_LBC_UPDATE_FHRS_str="( "$( printf "\"%s\" " "${EXTRN_MDL_LBC_UPDATE_FHRS[@]}" )")" -$SCRIPTSDIR/exregional_make_lbc1_to_lbcn.sh \ +$SCRIPTSDIR/exregional_make_lbcs.sh \ EXTRN_MDL_FNS="${EXTRN_MDL_FNS_str}" \ EXTRN_MDL_FILES_DIR="${EXTRN_MDL_FILES_DIR}" \ EXTRN_MDL_CDATE="${EXTRN_MDL_CDATE}" \ diff --git a/jobs/JREGIONAL_RUN_FV3 b/jobs/JREGIONAL_RUN_FCST similarity index 98% rename from jobs/JREGIONAL_RUN_FV3 rename to jobs/JREGIONAL_RUN_FCST index 38f75da12c..803d706099 100755 --- a/jobs/JREGIONAL_RUN_FV3 +++ b/jobs/JREGIONAL_RUN_FCST @@ -73,7 +73,7 @@ mkdir_vrfy -p ${CYCLE_DIR}/RESTART # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_fv3.sh \ +$SCRIPTSDIR/exregional_run_fcst.sh \ CYCLE_DIR="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/scripts/exregional_make_ic_lbc0.sh b/scripts/exregional_make_ics.sh similarity index 100% rename from scripts/exregional_make_ic_lbc0.sh rename to scripts/exregional_make_ics.sh diff --git a/scripts/exregional_make_lbc1_to_lbcn.sh b/scripts/exregional_make_lbcs.sh similarity index 100% rename from scripts/exregional_make_lbc1_to_lbcn.sh rename to scripts/exregional_make_lbcs.sh diff --git a/scripts/exregional_run_fv3.sh b/scripts/exregional_run_fcst.sh similarity index 100% rename from scripts/exregional_run_fv3.sh rename to scripts/exregional_run_fcst.sh diff --git a/ush/generate_FV3SAR_wflow.sh b/ush/generate_FV3SAR_wflow.sh index e7028089ea..3445ed008f 100755 --- a/ush/generate_FV3SAR_wflow.sh +++ b/ush/generate_FV3SAR_wflow.sh @@ -106,7 +106,7 @@ cp_vrfy ${TEMPLATE_XML_FP} ${WFLOW_XML_FP} # #----------------------------------------------------------------------- # -PROC_RUN_FV3="${NUM_NODES}:ppn=${NCORES_PER_NODE}" +PROC_RUN_FCST="${NUM_NODES}:ppn=${NCORES_PER_NODE}" FHR=( $( seq 0 1 ${FCST_LEN_HRS} ) ) i=0 @@ -148,7 +148,7 @@ set_file_param "${WFLOW_XML_FP}" "EXTRN_MDL_NAME_ICS" "${EXTRN_MDL_NAME_ICS}" set_file_param "${WFLOW_XML_FP}" "EXTRN_MDL_NAME_LBCS" "${EXTRN_MDL_NAME_LBCS}" set_file_param "${WFLOW_XML_FP}" "EXTRN_MDL_FILES_SYSBASEDIR_ICS" "${EXTRN_MDL_FILES_SYSBASEDIR_ICS}" set_file_param "${WFLOW_XML_FP}" "EXTRN_MDL_FILES_SYSBASEDIR_LBCS" "${EXTRN_MDL_FILES_SYSBASEDIR_LBCS}" -set_file_param "${WFLOW_XML_FP}" "PROC_RUN_FV3" "${PROC_RUN_FV3}" +set_file_param "${WFLOW_XML_FP}" "PROC_RUN_FCST" "${PROC_RUN_FCST}" set_file_param "${WFLOW_XML_FP}" "DATE_FIRST_CYCL" "${DATE_FIRST_CYCL}" set_file_param "${WFLOW_XML_FP}" "DATE_LAST_CYCL" "${DATE_LAST_CYCL}" set_file_param "${WFLOW_XML_FP}" "YYYY_FIRST_CYCL" "${YYYY_FIRST_CYCL}" diff --git a/ush/templates/FV3SAR_wflow.xml b/ush/templates/FV3SAR_wflow.xml index 316af64733..6d861dd0ad 100644 --- a/ush/templates/FV3SAR_wflow.xml +++ b/ush/templates/FV3SAR_wflow.xml @@ -61,7 +61,7 @@ Variables that are not modified by the workflow generation script. - + 00:10:00"> @@ -70,12 +70,12 @@ Variables that are not modified by the workflow generation script. 00:45:00"> 00:30:00"> 01:00:00"> -04:30:00"> +04:30:00"> 00:30:00"> -&QUEUE_DEFAULT;&ACCOUNT;"> -&QUEUE_HPSS;&ACCOUNT;"> -&QUEUE_FCST;&ACCOUNT;"> +&QUEUE_DEFAULT;&ACCOUNT;"> +&QUEUE_HPSS;&ACCOUNT;"> +&QUEUE_FCST;&ACCOUNT;"> - &RSRC_RUN_FV3; - &RSRV_RUN_FV3; + &RSRC_RUN_FCST; + &RSRV_RUN_FCST; - &JOBSDIR;/JREGIONAL_RUN_FV3 - &PROC_RUN_FV3; + &JOBSDIR;/JREGIONAL_RUN_FCST + &PROC_RUN_FCST; &TN_RUN_FCST; &LOGDIR;/&TN_RUN_FCST;_@Y@m@d@H.log From e898524bbe3ed8ffe6e99f4a59448374867961e6 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 18 Nov 2019 12:04:40 -0700 Subject: [PATCH 6/9] For simplicity and clarity, change variable name from APRUN_SFC to APRUN in scripts/exregional_make_sfc_climo.sh. --- scripts/exregional_make_sfc_climo.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index f0927dad67..37bee4c350 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -123,19 +123,19 @@ case $MACHINE in "WCOSS_C") # This could be wrong. Just a guess since I don't have access to this machine. - APRUN_SFC=${APRUN_SFC:-"aprun -j 1 -n 6 -N 6"} + APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} ;; "WCOSS") # This could be wrong. Just a guess since I don't have access to this machine. - APRUN_SFC=${APRUN_SFC:-"aprun -j 1 -n 6 -N 6"} + APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} ;; "THEIA") # Need to load intel/15.1.133. This and all other module loads should go into a module file. module load intel/15.1.133 module list - APRUN_SFC="mpirun -np ${SLURM_NTASKS}" + APRUN="mpirun -np ${SLURM_NTASKS}" ;; "HERA") @@ -149,14 +149,14 @@ case $MACHINE in module load esmflocal/8_0_48b.netcdf47 #module load esmf/7.1.0r module list - APRUN_SFC="srun" + APRUN="srun" ;; *) print_err_msg_exit "\ Run command has not been specified for this machine: MACHINE = \"$MACHINE\" - APRUN_SFC = \"$APRUN_SFC\"" + APRUN = \"$APRUN\"" ;; esac @@ -167,7 +167,7 @@ esac # #----------------------------------------------------------------------- # -$APRUN_SFC ${EXECDIR}/sfc_climo_gen || print_err_msg_exit "\ +$APRUN ${EXECDIR}/sfc_climo_gen || print_err_msg_exit "\ Call to executable that generates surface climatology files returned with nonzero exit code." # From 632c41e46f8587bab8f6d31ff04dc8196e9fe944 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 18 Nov 2019 14:12:33 -0700 Subject: [PATCH 7/9] Changes (for Hera only thus far) to move module files out of J-jobs and ex-scripts and into an individual module file for each task. Details below. For all tasks on Hera: 1) Remove all module purge/use/load commands in J-jobs and ex-scripts and place them in separate module files (one per task) in directory modulefiles/hera/. 2) Modify the rocoto XML template file so that it calls each J-job via the ush/load_modules_run_task.sh script. This script first loads the module file for the task and then runs the J-job using the exec command (which kills the load_modules_run_task.sh process and replaces it with the J-job, all while keeping the appropriate modules loaded in the environment). For the run_fcst task on Hera: 1) In ush/generate_FV3SAR_wflow.sh, remove copying of module-setup.sh.inc from "${NEMSfv3gfs_DIR}/NEMS/src/conf" to the experiment directory as module-setup.sh, since we now do the module setup directly in ush/load_modules_run_task.sh. Also, instead of adding a line into module-setup.sh that modifies LD_LIBRARY_PATH to add the CCPP library path, perform this modification in scripts/exregional_run_fcst.sh directly (i.e. in the shell, without modifying any files). 2) In ush/generate_FV3SAR_wflow.sh, remove copying of modules.fv3 from "${NEMSfv3gfs_DIR}/NEMS/src/conf" to the experiment directory. Instead, create a symlink at modulefiles/hera/run_fcst_ccpp that points to "${NEMSfv3gfs_DIR}/NEMS/src/conf/modules.fv3". Note that the NEMSfv3gfs code must first be built in order for this symlink's target to exist. 3) In scripts/exregional_run_fcst.sh, remove creation of links in cycle directory that point to module-setup.sh and modules.fv3 in the experiment directory. The latter two files no longer exist, and with the above changes, the links are no longer needed. 4) Make the name of the module file for the forecast task dependent on the USE_CCPP flag, since we use different module files depending on whether or not we're using CCPP: run_fcst_ccpp vs. run_fcst_no_ccpp. The second one is an actual file while the first is the symlink mentioned above. --- jobs/JREGIONAL_MAKE_ICS | 26 ---- jobs/JREGIONAL_MAKE_LBCS | 26 ---- modulefiles/hera/get_extrn_ics | 8 ++ modulefiles/hera/get_extrn_lbcs | 8 ++ modulefiles/hera/make_grid | 1 + modulefiles/hera/make_ics | 23 ++++ modulefiles/hera/make_lbcs | 23 ++++ modulefiles/hera/make_orog | 10 ++ modulefiles/hera/make_sfc_climo | 15 +++ modulefiles/hera/run_fcst_ccpp | 1 + modulefiles/hera/run_fcst_no_ccpp | 12 ++ modulefiles/hera/run_post | 30 +++++ scripts/exregional_get_extrn_files.sh | 10 -- scripts/exregional_make_orog.sh | 17 +-- scripts/exregional_make_sfc_climo.sh | 10 -- scripts/exregional_run_fcst.sh | 31 +---- scripts/exregional_run_post.sh | 34 +----- ush/generate_FV3SAR_wflow.sh | 81 ------------- ush/load_modules_run_task.sh | 164 +++++++++++++++++++++++--- ush/templates/FV3SAR_wflow.xml | 36 +++--- 20 files changed, 298 insertions(+), 268 deletions(-) create mode 100644 modulefiles/hera/get_extrn_ics create mode 100644 modulefiles/hera/get_extrn_lbcs create mode 100644 modulefiles/hera/make_ics create mode 100644 modulefiles/hera/make_lbcs create mode 100644 modulefiles/hera/make_orog create mode 100644 modulefiles/hera/make_sfc_climo create mode 120000 modulefiles/hera/run_fcst_ccpp create mode 100644 modulefiles/hera/run_fcst_no_ccpp create mode 100644 modulefiles/hera/run_post diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index eeaa95f43d..5614faaa27 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -119,35 +119,9 @@ case "$MACHINE" in ;; # "HERA") -# - { save_shell_opts; set +x; } > /dev/null 2>&1 - ulimit -s unlimited # ulimit -a - - module purge - module load intel/18.0.5.274 - module load impi/2018.0.4 - module load netcdf/4.6.1 - - module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles - module load esmf/8.0.0bs21 - - module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles - module load w3nco - module load nemsio/2.2.3 - module load bacio - module load sp - module load sfcio - module load sigio - - module load wgrib2 - - module list - APRUN="srun" - - { restore_shell_opts; } > /dev/null 2>&1 ;; # "JET") diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index e29919d6fe..24ff59da65 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -119,35 +119,9 @@ case "$MACHINE" in ;; # "HERA") -# - { save_shell_opts; set +x; } > /dev/null 2>&1 - ulimit -s unlimited # ulimit -a - - module purge - module load intel/18.0.5.274 - module load impi/2018.0.4 - module load netcdf/4.6.1 - - module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles - module load esmf/8.0.0bs21 - - module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles - module load w3nco - module load nemsio/2.2.3 - module load bacio - module load sp - module load sfcio - module load sigio - - module load wgrib2 - - module list - APRUN="srun" - - { restore_shell_opts; } > /dev/null 2>&1 ;; # diff --git a/modulefiles/hera/get_extrn_ics b/modulefiles/hera/get_extrn_ics new file mode 100644 index 0000000000..9935033fd2 --- /dev/null +++ b/modulefiles/hera/get_extrn_ics @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_ics task. +############################################################# + +module purge + +module load hpss + diff --git a/modulefiles/hera/get_extrn_lbcs b/modulefiles/hera/get_extrn_lbcs new file mode 100644 index 0000000000..1919f3355a --- /dev/null +++ b/modulefiles/hera/get_extrn_lbcs @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_lbcs task. +############################################################# + +module purge + +module load hpss + diff --git a/modulefiles/hera/make_grid b/modulefiles/hera/make_grid index 66368b2aaa..2d651a5c94 100644 --- a/modulefiles/hera/make_grid +++ b/modulefiles/hera/make_grid @@ -3,6 +3,7 @@ ############################################################# module purge + module load intel/18.0.5.274 module load netcdf/4.7.0 module load hdf5/1.10.5 diff --git a/modulefiles/hera/make_ics b/modulefiles/hera/make_ics new file mode 100644 index 0000000000..29098fd729 --- /dev/null +++ b/modulefiles/hera/make_ics @@ -0,0 +1,23 @@ +#%Module##################################################### +## Module file for make_ics task. +############################################################# + +module purge + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.6.1 + +module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles +module load esmf/8.0.0bs21 + +module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles +module load w3nco +module load nemsio/2.2.3 +module load bacio +module load sp +module load sfcio +module load sigio + +module load wgrib2 + diff --git a/modulefiles/hera/make_lbcs b/modulefiles/hera/make_lbcs new file mode 100644 index 0000000000..e464cd9fea --- /dev/null +++ b/modulefiles/hera/make_lbcs @@ -0,0 +1,23 @@ +#%Module##################################################### +## Module file for make_lbcs task. +############################################################# + +module purge + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.6.1 + +module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles +module load esmf/8.0.0bs21 + +module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles +module load w3nco +module load nemsio/2.2.3 +module load bacio +module load sp +module load sfcio +module load sigio + +module load wgrib2 + diff --git a/modulefiles/hera/make_orog b/modulefiles/hera/make_orog new file mode 100644 index 0000000000..295940dae9 --- /dev/null +++ b/modulefiles/hera/make_orog @@ -0,0 +1,10 @@ +#%Module##################################################### +## Module file for make_orog task. +############################################################# + +module purge + +module load intel/18.0.5.274 +module load netcdf/4.7.0 +module load hdf5/1.10.4 + diff --git a/modulefiles/hera/make_sfc_climo b/modulefiles/hera/make_sfc_climo new file mode 100644 index 0000000000..c83cc119b9 --- /dev/null +++ b/modulefiles/hera/make_sfc_climo @@ -0,0 +1,15 @@ +#%Module##################################################### +## Module file for make_sfc_climo task. +############################################################# + +module purge + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.6.1 + +#NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +#module use -a $NCEPLIBS/modulefiles +module use -a /scratch1/NCEPDEV/global/gwv/l819/lib/modulefiles +module load esmflocal/8_0_48b.netcdf47 + diff --git a/modulefiles/hera/run_fcst_ccpp b/modulefiles/hera/run_fcst_ccpp new file mode 120000 index 0000000000..eacd4ef1dc --- /dev/null +++ b/modulefiles/hera/run_fcst_ccpp @@ -0,0 +1 @@ +../../sorc/NEMSfv3gfs/NEMS/src/conf/modules.nems \ No newline at end of file diff --git a/modulefiles/hera/run_fcst_no_ccpp b/modulefiles/hera/run_fcst_no_ccpp new file mode 100644 index 0000000000..290aff9daf --- /dev/null +++ b/modulefiles/hera/run_fcst_no_ccpp @@ -0,0 +1,12 @@ +#%Module##################################################### +## Module file for run_fcst task without CCPP. +############################################################# + +module purge + +module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.6.1 +module load pnetcdf/1.10.0 + diff --git a/modulefiles/hera/run_post b/modulefiles/hera/run_post new file mode 100644 index 0000000000..70df4e2e82 --- /dev/null +++ b/modulefiles/hera/run_post @@ -0,0 +1,30 @@ +#%Module##################################################### +## Module file for run_post task. +############################################################# + +module purge + +module load intel/19.0.4.243 +module load impi/2019.0.4 + +# Load nceplibs modules. +module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles + +module load sigio/2.1.1 +module load jasper/1.900.1 +module load png/1.2.44 +module load z/1.2.11 +module load sfcio/1.1.1 +module load nemsio/2.2.4 +module load bacio/2.0.3 +module load g2/3.1.1 +module load gfsio/1.1.0 +module load ip/3.0.2 +module load sp/2.0.3 +module load w3emc/2.3.1 +module load w3nco/2.0.7 +module load crtm/2.2.5 +module load netcdf/4.7.0 +module load g2tmpl/1.5.1 +module load wrfio/1.1.1 + diff --git a/scripts/exregional_get_extrn_files.sh b/scripts/exregional_get_extrn_files.sh index 635984c34b..49ddec8657 100755 --- a/scripts/exregional_get_extrn_files.sh +++ b/scripts/exregional_get_extrn_files.sh @@ -252,16 +252,6 @@ will be copied (EXTRN_MDL_FILES_DIR) are: # #----------------------------------------------------------------------- # -# Load necessary modules. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set +x; } > /dev/null 2>&1 - module load hpss - { restore_shell_opts; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# # Get the number of archive files to consider. # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index d1912da1e7..c38c41d9f6 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -179,23 +179,10 @@ case $MACHINE in "HERA") -# - { save_shell_opts; set +x; } > /dev/null 2>&1 - - . /apps/lmod/lmod/init/sh - module purge - module load intel/18.0.5.274 - module load netcdf/4.7.0 - module load hdf5/1.10.4 - module list - - { restore_shell_opts; } > /dev/null 2>&1 - - export APRUN="time" - export topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog" - ulimit -s unlimited ulimit -a + export APRUN="time" + export topo_dir="/scratch1/NCEPDEV/global/glopara/fix/fix_orog" ;; diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 37bee4c350..498d45402e 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -139,16 +139,6 @@ case $MACHINE in ;; "HERA") - module purge - module load intel/18.0.5.274 - module load impi/2018.0.4 - module load netcdf/4.6.1 - #module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles - export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib - module use -a $NCEPLIBS/modulefiles - module load esmflocal/8_0_48b.netcdf47 - #module load esmf/7.1.0r - module list APRUN="srun" ;; diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 7cf0a46285..1286430dbd 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -144,37 +144,10 @@ case $MACHINE in ;; # "HERA") -# - - if [ "${USE_CCPP}" = "TRUE" ]; then - -# Need to change to the experiment directory to correctly load necessary -# modules for CCPP-version of FV3SAR in lines below - cd_vrfy ${EXPTDIR} - - set +x - source ./module-setup.sh - module use $( pwd -P ) - module load modules.fv3 - module list - set -x - - else - - . /apps/lmod/lmod/init/sh - module purge - module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles - module load intel/18.0.5.274 - module load impi/2018.0.4 - module load netcdf/4.6.1 - module load pnetcdf/1.10.0 - module list - - fi - ulimit -s unlimited ulimit -a APRUN="srun" + LD_LIBRARY_PATH="${NEMSfv3gfs_DIR}/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ;; # "JET") @@ -414,8 +387,6 @@ ln_vrfy -sf -t ${CYCLE_DIR} $EXPTDIR/${FIELD_TABLE_FN} ln_vrfy -sf -t ${CYCLE_DIR} $EXPTDIR/${NEMS_CONFIG_FN} if [ "${USE_CCPP}" = "TRUE" ]; then - ln_vrfy -sf -t ${CYCLE_DIR} $EXPTDIR/module-setup.sh - ln_vrfy -sf -t ${CYCLE_DIR} $EXPTDIR/modules.fv3 if [ "${CCPP_PHYS_SUITE}" = "GSD" ]; then ln_vrfy -sf -t ${CYCLE_DIR} $EXPTDIR/suite_FV3_GSD_v0.xml elif [ "${CCPP_PHYS_SUITE}" = "GFS" ]; then diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 50f1d5f28c..79ab2e47cb 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -122,39 +122,7 @@ case $MACHINE in "HERA") - { save_shell_opts; set +x; } > /dev/null 2>&1 - module purge - - module load intel/19.0.4.243 - module load impi/2019.0.4 - -# module use /contrib/modulefiles - module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles - -# Loading nceplibs modules - module load sigio/2.1.1 - module load jasper/1.900.1 - module load png/1.2.44 - module load z/1.2.11 - module load sfcio/1.1.1 - module load nemsio/2.2.4 - module load bacio/2.0.3 - module load g2/3.1.1 -# module load xmlparse/v2.0.0 - module load gfsio/1.1.0 - module load ip/3.0.2 - module load sp/2.0.3 - module load w3emc/2.3.1 - module load w3nco/2.0.7 - module load crtm/2.2.5 -# module load netcdf/3.6.3 - module load netcdf/4.7.0 - module load g2tmpl/1.5.1 - module load wrfio/1.1.1 - - export NDATE=/scratch3/NCEPDEV/nwprod/lib/prod_util/v1.1.0/exec/ndate - - { restore_shell_opts; } > /dev/null 2>&1 +# export NDATE=/scratch3/NCEPDEV/nwprod/lib/prod_util/v1.1.0/exec/ndate APRUN="srun" ;; diff --git a/ush/generate_FV3SAR_wflow.sh b/ush/generate_FV3SAR_wflow.sh index 3445ed008f..1c9efead09 100755 --- a/ush/generate_FV3SAR_wflow.sh +++ b/ush/generate_FV3SAR_wflow.sh @@ -503,91 +503,10 @@ Copying templates of various input files to the experiment directory..." # # If using CCPP... # -# If USE_CCPP is set to "TRUE", copy the appropriate modulefile, the -# CCPP physics suite definition file (an XML file), and possibly other -# suite-dependent files to the experiment directory. -# -# The modulefile modules.nems in the directory -# -# $NEMSfv3gfs_DIR/NEMS/src/conf -# -# is generated during the FV3 build process and this is configured pro- -# perly for the machine, shell environment, etc. Thus, we can just copy -# it to the experiment directory without worrying about what machine -# we're on, but this still needs to be confirmed. -# -# Note that a modulefile is a file whose first line is the "magic coo- -# kie" '#%Module'. It is interpreted by the "module load ..." command. -# It sets environment variables (including prepending/appending to -# paths) and loads modules. -# -# QUESTION: -# Why don't we do this for the non-CCPP version of FV3? -# -# ANSWER: -# Because for that case, we load different versions of intel and impi -# (compare modules.nems to the modules loaded for the case of USE_CCPP -# set to "FALSE" in run_FV3SAR.sh). Maybe these can be combined at some -# point. Note that a modules.nems file is generated in the same rela- -# tive location in the non-CCPP-enabled version of NEMSfv3gfs, so maybe -# that can be used and the run_FV3SAR.sh script modified to accomodate -# such a change. That way the below can be performed for both the CCPP- -# enabled and non-CCPP-enabled versions of NEMSfv3gfs. -# #----------------------------------------------------------------------- # if [ "${USE_CCPP}" = "TRUE" ]; then # -# Copy the shell script that initializes the Lmod (Lua-based module) -# system/software for handling modules. This script: -# -# 1) Detects the shell in which it is being invoked (i.e. the shell of -# the "parent" script in which it is being sourced). -# 2) Detects the machine it is running on and and calls the appropriate -# (shell- and machine-dependent) initalization script to initialize -# Lmod. -# 3) Purges all modules. -# 4) Uses the "module use ..." command to prepend or append paths to -# Lmod's search path (MODULEPATH). -# - print_info_msg "$VERBOSE" " -Copying the shell script that initializes the Lmod (Lua-based module) -system/software for handling modules..." -# -# The following might have to be made shell-dependent, e.g. if using csh -# or tcsh, copy over the file module-setup.csh.inc??. -# -# It may be convenient to also copy over this script when running the -# non-CCPP version of the FV3SAR and try to simplify the run script -# (run_FV3SAR.sh) so that it doesn't depend on whether USE_CCPP is set -# to "TRUE" or "FALSE". We can do that, but currently the non-CCPP and -# CCPP-enabled versions of the FV3SAR code use different versions of -# intel and impi, so module-setup.sh must account for this. -# - cp_vrfy ${NEMSfv3gfs_DIR}/NEMS/src/conf/module-setup.sh.inc \ - $EXPTDIR/module-setup.sh -# -# Append the command that adds the path to the CCPP libraries (via the -# shell variable LD_LIBRARY_PATH) to the Lmod initialization script in -# the experiment directory. This is needed if running the dynamic build -# of the CCPP-enabled version of the FV3SAR. -# - { cat << EOM >> $EXPTDIR/module-setup.sh -# -# Add path to libccpp.so and libccpphys.so to LD_LIBRARY_PATH" -# -export LD_LIBRARY_PATH="${NEMSfv3gfs_DIR}/ccpp/lib\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append command to add path to CCPP libraries to -the Lmod initialization script in the experiment directory returned with -a nonzero status." - - print_info_msg "$VERBOSE" " -Copying the modulefile required for running the CCPP-enabled version of -the FV3SAR under NEMS to the experiment directory..." - cp_vrfy ${NEMSfv3gfs_DIR}/NEMS/src/conf/modules.nems $EXPTDIR/modules.fv3 -# #----------------------------------------------------------------------- # # If using CCPP with the GFS physics suite... diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index d8c8434639..eeb7d4d58c 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -66,8 +66,34 @@ fi # #----------------------------------------------------------------------- # -# Call the script that defines the module() function. This is needed so -# we can perform "module load ..." calls later below. +# Source the script that initializes the Lmod (Lua-based module) system/ +# software for handling modules. This script defines the module() and +# other functions. These are needed so we can perform the "module use +# ..." and "module load ..." calls later below that are used to load the +# appropriate module file for the specified task. +# +# Note that the build of NEMSfv3gfs generates the shell script at +# +# ${NEMSfv3gfs_DIR}/NEMS/src/conf/module-setup.sh +# +# that can be used to initialize the Lmod (Lua-based module) system/ +# software for handling modules. This script: +# +# 1) Detects the shell in which it is being invoked (i.e. the shell of +# the "parent" script in which it is being sourced). +# 2) Detects the machine it is running on and and calls the appropriate +# (shell- and machine-dependent) initalization script to initialize +# Lmod. +# 3) Purges all modules. +# 4) Uses the "module use ..." command to prepend or append paths to +# Lmod's search path (MODULEPATH). +# +# We could use this module-setup.sh script to initialize Lmod, but since +# it is only found in the NEMSfv3gfs code, here we prefer to perform our +# own initialization. Ideally, there should be one module-setup.sh +# script that is used by all external repos/codes, but such a script +# does not exist. If/when it does, we will consider switching to it in- +# stead of using the case-statement below. # #----------------------------------------------------------------------- # @@ -94,7 +120,7 @@ case "$MACHINE" in ;; # *) - print_err_msg_exit " + print_err_msg_exit "\ The script to source to initialize lmod (module loads) has not yet been specified for the current machine (MACHINE): MACHINE = \"$MACHINE\"" @@ -104,11 +130,7 @@ esac # #----------------------------------------------------------------------- # -# Get the task name. Then shift the argument list so that the firs ar- -# gument (the task name) gets dropped from the start of the arguments -# list. This results in the shell variable $@ containing only the re- -# maining arguments (which in this case should consist of the full path -# to the J-job to call). +# Get the task name and the name of the J-job script. # #----------------------------------------------------------------------- # @@ -117,29 +139,133 @@ jjob_fp="$2" # #----------------------------------------------------------------------- # -# Purge modules and load the module file for the specified task on the -# current machine. Note that the path to this module file is given by +# Set the directory (modules_dir) in which the module files for the va- +# rious workflow tasks are located. Also, set the name of the module +# file for the specified task. +# +# A module file is a file whose first line is the "magic cookie" string +# '#%Module'. It is interpreted by the "module load ..." command. It +# sets environment variables (including prepending/appending to paths) +# and loads modules. +# +# The regional_workflow repository contains module files for all the +# workflow tasks in the template rocoto XML file for the FV3SAR work- +# flow. The full path to a module file for a given task is +# +# $HOMErrfs/modulefiles/$machine/${task_name} +# +# where HOMErrfs is the base directory of the workflow, machine is the +# name of the machine that we're running on (in lowercase), and task_- +# name is the name of the current task (an input to this script). For +# all tasks in the rocoto XML except run_fcst, these are actual files +# (as opposed to symlinks). For the run_fcst task, there are two possi- +# ble module files. The first one is named "run_fcst_no_ccpp" and is +# used to run FV3 without CCPP (i.e. it is used if USE_CCPP is set to +# "FALSE" in the experiment/workflow configuration file). This is also +# an actual file. The second one is named "run_fcst_ccpp" and is used +# to run FV3 with CCPP (i.e. it is used if USE_CCPP is set to "TRUE"). +# This second file is a symlink (and is a part of the regional_workflow +# repo), and its target is +# +# ${NEMSfv3gfs_DIR}/NEMS/src/conf/modules.fv3 +# +# Here, NEMSfv3gfs_DIR is the directory in which the NEMSfv3gfs reposi- +# tory containing the FV3 model is cloned (normally "$HOMErrfs/sorc/ +# NEMSfv3gfs"), and modules.fv3 is a module file that is generated by +# the NEMSfv3gfs build process. It contains the appropriate modules to +# use when running the FV3 model. Thus, we just point to it via the +# symlink "run_fcst_ccpp" in the modulefiles/$machine directory. +# +# QUESTION: +# Why don't we do this for the non-CCPP version of FV3? +# +# ANSWER: +# Because for that case, we load different versions of intel and impi +# (compare modules.nems to the modules loaded for the case of USE_CCPP +# set to "FALSE" in run_FV3SAR.sh). Maybe these can be combined at some +# point. Note that a modules.nems file is generated in the same rela- +# tive location in the non-CCPP-enabled version of NEMSfv3gfs, so maybe +# that can be used and the run_FV3SAR.sh script modified to accomodate +# such a change. That way the below can be performed for both the CCPP- +# enabled and non-CCPP-enabled versions of NEMSfv3gfs. # -# $HOMErrfs/modulefiles/$machine/${task_name} +#----------------------------------------------------------------------- # -# where HOMErrfs is the workflow home directory, machine is the name of -# the current machine in lowercase, and task_name is the name of the -# task that this script will launch (via the exec command below). +machine=${MACHINE,,} +modules_dir="$HOMErrfs/modulefiles/$machine" +modulefile_name="${task_name}" + +if [ "${task_name}" = "run_fcst" ]; then + if [ "${USE_CCPP}" = "TRUE" ]; then + modulefile_name=${modulefile_name}_ccpp + else + modulefile_name=${modulefile_name}_no_ccpp + fi +fi +# +#----------------------------------------------------------------------- +# +# Use the "readlink" command to resolve the full path to the module file +# and then verify that the file exists. This is not necessary for most +# tasks, but for the run_fcst task, when CCPP is enabled, the module +# file in the modules directory is not a regular file but a symlink to a +# file in the NEMSfv3gfs external repo. This latter target file will +# exist only if the NEMSfv3gfs code has already been built. Thus, we +# now check to make sure that the module file exits. +# +#----------------------------------------------------------------------- +# +modulefile_path=$( readlink -f "${modules_dir}/${modulefile_name}" ) + +if [ ! -f "${modulefile_path}" ]; then + + if [ "${task_name}" = "run_fcst" ]; then + + print_err_msg_exit "\ +The target (modulefile_path) of the symlink (modulefile_name) in the +task modules directory (modules_dir) that points to module file for this +task (task_name) does not exsit: + task_name = \"${task_name}\" + modulefile_name = \"${modulefile_name}\" + modules_dir = \"${modules_dir}\" + modulefile_path = \"${modulefile_path}\" +This is likely because the forecast model code (NEMSfv3gfs) has not yet +been built. +" + + else + + print_err_msg_exit "\ +The module file (modulefile_path) specified for this task (task_name) +does not exist: + task_name = \"${task_name}\" + modulefile_path = \"${modulefile_path}\"" + +fi +# +#----------------------------------------------------------------------- +# +# Purge modules and load the module file for the specified task on the +# current machine. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " Loading modules for task \"${task_name}\" ..." - -machine=${MACHINE,,} + module purge -module use $HOMErrfs/modulefiles/$machine -module load ${task_name} + +module use "${modules_dir}" || print_err_msg_exit "\ +Call to \"module use\" command failed." + +module load ${modulefile_name} || print_err_msg_exit "\ +Call to \"module load\" command failed." + module list # #----------------------------------------------------------------------- # -# Use the exec ocmmand to terminate the current script and launch the +# Use the exec command to terminate the current script and launch the # J-job for the specified task. # #----------------------------------------------------------------------- diff --git a/ush/templates/FV3SAR_wflow.xml b/ush/templates/FV3SAR_wflow.xml index 6d861dd0ad..0febda832f 100644 --- a/ush/templates/FV3SAR_wflow.xml +++ b/ush/templates/FV3SAR_wflow.xml @@ -86,15 +86,15 @@ killing itelf off) using the exec command. - - - - - - - - - + + + + + + + + + ]> @@ -104,7 +104,7 @@ Task names. &DATE_FIRST_CYCL;CC00 &DATE_LAST_CYCL;CC00 24:00:00 - &LOGDIR;/FV3_wflow.log + &LOGDIR;/FV3SAR_wflow.log