From 3965b56957896565c0469214b3d99c4d4e69a409 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 03:39:52 +0000 Subject: [PATCH] Combine CI infrastructure, use get_expts_status etc to simplify Jenkins. --- .cicd/Jenkinsfile | 7 +- .cicd/scripts/srw_build.sh | 20 +- .cicd/scripts/srw_test.sh | 223 ++---------------- test/build.sh | 129 +++++----- tests/WE2E/get_expts_status.sh | 1 + tests/WE2E/machine_suites/comprehensive | 58 +++++ tests/WE2E/machine_suites/custom | 1 + .../{fundamental.txt => fundamental} | 0 tests/WE2E/setup_WE2E_tests.sh | 25 +- 9 files changed, 174 insertions(+), 290 deletions(-) create mode 100644 tests/WE2E/machine_suites/comprehensive create mode 100644 tests/WE2E/machine_suites/custom rename tests/WE2E/machine_suites/{fundamental.txt => fundamental} (100%) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 79174c4d20..9ac3361504 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -69,7 +69,7 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'hera' // 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -97,6 +97,7 @@ pipeline { BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_') BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}" BUILD_NAME = "ufs-srweather-app_${env.BUILD_VERSION}" + INSTALL_NAME = "install_${env.SRW_COMPILER}" } stages { @@ -127,7 +128,7 @@ pipeline { post { success { - sh 'tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" bin include lib share' + sh 'tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" "${INSTALL_NAME}"' s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.log", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } @@ -140,7 +141,7 @@ pipeline { } environment { - SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/experiments" + SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/../expt_dirs" } steps { diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh index fed4170e3a..e55c3f2fc5 100755 --- a/.cicd/scripts/srw_build.sh +++ b/.cicd/scripts/srw_build.sh @@ -24,17 +24,13 @@ else platform="${SRW_PLATFORM}" fi -build_dir="${workspace}/build" +# Build and install +cd ${workspace}/test +./build.sh ${platform} ${SRW_COMPILER} +cd - -# Set build related environment variables and load required modules. -source "${workspace}/etc/lmod-setup.sh" "${platform}" -module use "${workspace}/modulefiles" -module load "build_${platform}_${SRW_COMPILER}" +# Create combined log file for upload to s3 +build_dir="${workspace}/build_${SRW_COMPILER}" +cat ${build_dir}/log.cmake ${build_dir}/log.make \ + >${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log -# Compile SRW application and install to repository root. -mkdir "${build_dir}" -pushd "${build_dir}" - build_log_file="${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log" - cmake -DCMAKE_INSTALL_PREFIX="${workspace}" "${workspace}" | tee "${build_log_file}" - make -j "${MAKE_JOBS}" | tee --append "${build_log_file}" -popd diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh index 0582815e74..26b5536f22 100755 --- a/.cicd/scripts/srw_test.sh +++ b/.cicd/scripts/srw_test.sh @@ -25,219 +25,42 @@ else platform="${SRW_PLATFORM}" fi -declare we2e_experiment_base_dir -if [[ -n "${SRW_WE2E_EXPERIMENT_BASE_DIR}" ]]; then - we2e_experiment_base_dir="${SRW_WE2E_EXPERIMENT_BASE_DIR}" -else - we2e_experiment_base_dir="${workspace}/experiments" -fi - +# Test directories +we2e_experiment_base_dir="${workspace}/../expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" -we2e_test_file="${we2e_test_dir}/experiments.txt" - -# The fundamental set of end-to-end tests to run. -declare -a we2e_fundamental_tests -we2e_fundamental_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta' - 'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR' - 'grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR' - 'grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta' - 'grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR' - 'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR' - 'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta' - 'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR' - 'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta' - 'nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR' - 'community_ensemble_2mems' - 'custom_ESGgrid' - 'deactivate_tasks' - 'inline_post' - 'nco_ensemble' - 'specify_DOT_OR_USCORE' - 'specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE' - 'specify_RESTART_INTERVAL' - 'specify_template_filenames') - -if [[ "${platform}" != 'gaea' && "${platform}" != 'noaacloud' ]]; then - we2e_fundamental_tests+=('MET_ensemble_verification' - 'MET_verification' - 'pregen_grid_orog_sfc_climo') -fi - -# The comprehensive set of end-to-end tests to run. -declare -a we2e_comprehensive_tests -we2e_comprehensive_tests=('community_ensemble_008mems' - 'custom_GFDLgrid' - 'custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE' - 'custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100' - 'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000' - 'get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS' - 'get_from_HPSS_ics_HRRR_lbcs_RAP' - 'get_from_HPSS_ics_RAP_lbcs_RAP' - 'get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio' - 'grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2' - 'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR' - 'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta' - 'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp' - 'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional' - 'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2' - 'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR' - 'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp' - 'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2' - 'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16' - 'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km' - 'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2' - 'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR' - 'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta' - 'grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR' - 'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta' - 'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta' - 'grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2' - 'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR' - 'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta' - 'grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta' - 'grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta' - 'grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2' - 'nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' - 'nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km') - -declare -a we2e_tests -we2e_tests=("${we2e_fundamental_tests[@]}") +# Run the end-to-end tests. if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then - we2e_tests+=("${we2e_comprehensive_tests[@]}") - - # Add additional tests for Hera. - if [[ "${platform}" == 'hera' ]]; then - we2e_tests+=('specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS') - fi -fi - -# Parses the test log for the status of a specific test. -function workflow_status() { - local test="$1" - - local test_dir="${we2e_experiment_base_dir}/${test}" - local log_file="${test_dir}/log.launch_FV3LAM_wflow" - - if [[ -f "${log_file}" ]]; then - local status - status="$(awk 'BEGIN {FS=":";} $1 ~ "^[[:space:]]+Workflow status" {print $2}' "${log_file}" |\ - tail -1 |\ - sed --regexp-extended --expression 's/^[[:space:]]*(.*)[[:space:]]*$/\1/')" - if [[ "${status}" == 'IN PROGRESS' || "${status}" == 'SUCCESS' || "${status}" == 'FAILURE' ]]; then - echo "${status}" - else - echo 'UNKNOWN' - fi - else - echo 'NOT FOUND' - fi -} - -# Gets the status of all tests. Prints the number of tests that are running. -# Returns a non-zero code when all tests reach a final state. -function check_progress() { - local in_progress=false - local remaining=0 - - for test in "${we2e_tests[@]}"; do - local status - status="$(workflow_status "${test}")" - if [[ "${status}" == 'IN PROGRESS' ]]; then - in_progress=true - (( remaining++ )) - fi - done - - if "${in_progress}"; then - echo "Tests remaining: ${remaining}" - else - return 1 - fi -} - -# Prints the status of all tests. -function get_results() { - for test in "${we2e_tests[@]}"; do - local status - status="$(workflow_status "${test}")" - echo "${test} ${status}" - done -} - -# Verify that there is a non-zero sized weather model executable. -[[ -s "${workspace}/bin/ufs_model" ]] || [[ -s "${workspace}/bin/NEMS.exe" ]] - -# Set test related environment variables and load required modules. -source "${workspace}/etc/lmod-setup.sh" "${platform}" -module use "${workspace}/modulefiles" -module load "build_${platform}_${SRW_COMPILER}" -module load "wflow_${platform}" - -if [[ "${platform}" == 'cheyenne' ]]; then - export PATH="/glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427/bin:${PATH}" + test_type="comprehensive" else - if [[ "${platform}" == 'noaacloud' && -z "${PROJ_LIB-}" ]]; then - PROJ_LIB='' - fi - - conda activate regional_workflow + test_type="fundamental" fi -# Create the experiments/tests base directory. -mkdir "${we2e_experiment_base_dir}" - -# Generate the experiments/tests file. -for test in "${we2e_tests[@]}"; do - echo "${test}" >> "${we2e_test_file}" -done - -# Run the end-to-end tests. -"${we2e_test_dir}/run_WE2E_tests.sh" \ - tests_file="${we2e_test_file}" \ - machine="${platform}" \ - account="${SRW_PROJECT}" \ - expt_basedir="${we2e_experiment_base_dir}" \ - compiler="${SRW_COMPILER}" +cd ${we2e_test_dir} +./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} #${test_type} # Allow the tests to start before checking for status. # TODO: Create a parameter that sets the initial start delay. sleep 180 # Wait for all tests to complete. -while check_progress; do +while true; do + # Check status of all experiments + progress_file="${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt" + ./get_expts_status.sh expts_basedir="${we2e_experiment_base_dir}" \ + verbose="FALSE" num_log_lines=24000 | tee ${progress_file} + + # Exit loop only if there are not tests in progress + set +e + grep -q "Workflow status: IN PROGRESS" ${progress_file} + exit_code=$? + set -e + + if [[ $exit_code -ne 0 ]]; then + break + fi + # TODO: Create a paremeter that sets the poll frequency. sleep 60 done -# Get test results and write to a file. -results="$(get_results |\ - tee "${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt")" - -# Check that the number of tests equals the number of successes, otherwise -# exit with a non-zero code that equals the difference. -successes="$(awk '$2 == "SUCCESS" {print $1}' <<< "${results}" | wc -l)" -exit "$(( ${#we2e_tests[@]} - ${successes} ))" diff --git a/test/build.sh b/test/build.sh index cdf8d4e29b..7080b29638 100755 --- a/test/build.sh +++ b/test/build.sh @@ -7,7 +7,7 @@ # # Usage: see function usage below # -# Examples: ./build.sh hera >& test.out & +# Examples: ./build.sh hera intel all>& test.out & # set -eux # Uncomment for debugging #======================================================================= @@ -16,15 +16,14 @@ fail() { echo -e "\n$1\n" >> ${TEST_OUTPUT} && exit 1; } function usage() { echo - echo "Usage: $0 machine" + echo "Usage: $0 machine [compiler] [all/default]" echo exit 1 } machines=( hera jet cheyenne orion wcoss2 gaea odin singularity macos noaacloud ) -[[ $# -eq 2 ]] && usage - +[[ $# -gt 4 ]] && usage #----------------------------------------------------------------------- # Set some directories @@ -41,6 +40,10 @@ printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 machine=$(echo "${MACHINE}" | tr '[A-Z]' '[a-z]') # scripts in sorc need lower case machine name +compiler=${2:-"intel"} + +components=${3:-"default"} + #----------------------------------------------------------------------- # Check that machine is valid #----------------------------------------------------------------------- @@ -51,18 +54,6 @@ else exit 1 fi -#----------------------------------------------------------------------- -# Set compilers to be tested depending on machine -#----------------------------------------------------------------------- -if [ "${machine}" == "cheyenne" ] ; then - compilers=( intel gnu ) -elif [ "${machine}" == "macos" ] || [ "${machine}" == "singularity" ] ; then - compilers=( gnu ) -else - compilers=( intel ) -fi - - build_it=0 # Set to 1 to skip build (for testing pass/fail criteria) #----------------------------------------------------------------------- # Create the output file if it doesn't exist @@ -99,63 +90,63 @@ declare -a executables_created=( chgres_cube \ vcoord_gen ) -#----------------------------------------------------------------------- -# Array of all optional GSI executables built -#----------------------------------------------------------------------- -declare -a executables_gsi_created=( enkf.x \ - gsi.x \ - nc_diag_cat.x \ - ncdiag_cat_serial.x \ - test_nc_unlimdims.x ) - -#----------------------------------------------------------------------- -# Array of all optional rrfs_utl executables built -#----------------------------------------------------------------------- -declare -a executables_rrfs_utl_created=( adjust_soiltq.exe \ - check_imssnow_fv3lam.exe \ - fv3lam_nonvarcldana.exe \ - gen_annual_maxmin_GVF.exe \ - gen_cs.exe \ - gen_ensmean_recenter.exe \ - lakesurgery.exe \ - process_imssnow_fv3lam.exe \ - process_larccld.exe \ - process_Lightning.exe \ - process_metarcld.exe \ - process_NSSL_mosaic.exe \ - process_updatesst.exe \ - ref2tten.exe \ - update_bc.exe \ - update_GVF.exe \ - update_ice.exe \ - use_raphrrr_sfc.exe ) +if [ $components = "all" ]; then + #----------------------------------------------------------------------- + # Array of all optional GSI executables built + #----------------------------------------------------------------------- + executables_created+=( enkf.x \ + gsi.x \ + nc_diag_cat.x \ + ncdiag_cat_serial.x \ + test_nc_unlimdims.x ) + + #----------------------------------------------------------------------- + # Array of all optional rrfs_utl executables built + #----------------------------------------------------------------------- + executables_created=( adjust_soiltq.exe \ + check_imssnow_fv3lam.exe \ + fv3lam_nonvarcldana.exe \ + gen_annual_maxmin_GVF.exe \ + gen_cs.exe \ + gen_ensmean_recenter.exe \ + lakesurgery.exe \ + process_imssnow_fv3lam.exe \ + process_larccld.exe \ + process_Lightning.exe \ + process_metarcld.exe \ + process_NSSL_mosaic.exe \ + process_updatesst.exe \ + ref2tten.exe \ + update_bc.exe \ + update_GVF.exe \ + update_ice.exe \ + use_raphrrr_sfc.exe ) +fi #----------------------------------------------------------------------- # Set up the build environment and run the build script. #----------------------------------------------------------------------- - for compiler in "${compilers[@]}"; do - BUILD_DIR=${TOP_DIR}/build_${compiler} - INSTALL_DIR=${TOP_DIR}/install_${compiler} - EXEC_DIR=${INSTALL_DIR}/exec - if [ $build_it -eq 0 ] ; then - ./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${INSTALL_DIR} \ - --remove all || fail "Build ${machine} ${compiler} FAILED" - fi # End of skip build for testing - - #----------------------------------------------------------------------- - # check for existence of executables. - #----------------------------------------------------------------------- - n_fail=0 - for file in "${executables_created[@]}" "${executables_gsi_created[@]}" "${executables_rrfs_utl_created[@]}" ; do - exec_file=${EXEC_DIR}/${file} - if [ -f ${exec_file} ]; then - echo "SUCCEED: ${compiler} executable file ${exec_file} exists" >> ${TEST_OUTPUT} - else - echo "FAIL: ${compiler} executable file ${exec_file} does NOT exist" >> ${TEST_OUTPUT} - let "n_fail=n_fail+1" - fi - done -done # End compiler loop +BUILD_DIR=${TOP_DIR}/build_${compiler} +INSTALL_DIR=${TOP_DIR}/install_${compiler} +EXEC_DIR=${INSTALL_DIR}/exec +if [ $build_it -eq 0 ] ; then + ./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${INSTALL_DIR} \ + --remove ${components} || fail "Build ${machine} ${compiler} FAILED" +fi # End of skip build for testing + +#----------------------------------------------------------------------- +# check for existence of executables. +#----------------------------------------------------------------------- +n_fail=0 +for file in "${executables_created[@]}" ; do + exec_file=${EXEC_DIR}/${file} + if [ -f ${exec_file} ]; then + echo "SUCCEED: ${compiler} executable file ${exec_file} exists" >> ${TEST_OUTPUT} + else + echo "FAIL: ${compiler} executable file ${exec_file} does NOT exist" >> ${TEST_OUTPUT} + let "n_fail=n_fail+1" + fi +done #----------------------------------------------------------------------- # Set message for output #----------------------------------------------------------------------- diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index f9431bc529..65b742aa61 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -288,6 +288,7 @@ must be checked." # Source the variable definitions file. # . "./${var_defns_fn}" + export DEBUG="FALSE" # # If the workflow variable EXPT_SUBDIR is the same as the name of the # current subdirectory, then assume this subdirectory contains an active diff --git a/tests/WE2E/machine_suites/comprehensive b/tests/WE2E/machine_suites/comprehensive new file mode 100644 index 0000000000..14354eaf37 --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive @@ -0,0 +1,58 @@ +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +custom_ESGgrid +deactivate_tasks +inline_post +specify_DOT_OR_USCORE +specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE +specify_RESTART_INTERVAL +specify_template_filenames +community_ensemble_2mems +community_ensemble_008mems +custom_GFDLgrid +custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE +custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE +grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta +grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 +nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km diff --git a/tests/WE2E/machine_suites/custom b/tests/WE2E/machine_suites/custom new file mode 100644 index 0000000000..92d0d47a1b --- /dev/null +++ b/tests/WE2E/machine_suites/custom @@ -0,0 +1 @@ +custom_GFDLgrid diff --git a/tests/WE2E/machine_suites/fundamental.txt b/tests/WE2E/machine_suites/fundamental similarity index 100% rename from tests/WE2E/machine_suites/fundamental.txt rename to tests/WE2E/machine_suites/fundamental diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 1605582a86..dc747de113 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -25,10 +25,12 @@ function usage { echo - echo "Usage: $0 machine slurm_account | -h" + echo "Usage: $0 machine slurm_account [compiler] [test_type] | -h" echo echo " machine [required] is one of: ${machines[@]}" echo " slurm_account [required] case sensitive name of the user-specific slurm account" + echo " compiler [optional] compiler used for build" + echo " test_type [optional] test type: fundamental or comprehensive or any other name" echo " -h display this help" echo exit 1 @@ -42,8 +44,9 @@ if [ "$1" = "-h" ] ; then usage ; fi machine=$1 machine=$(echo "${machine}" | tr '[A-Z]' '[a-z]') # scripts in sorc need lower case machine name - account=$2 +compiler=${3:-intel} +test_type=${4:-custom} #----------------------------------------------------------------------- # Set directories @@ -62,16 +65,16 @@ EXPTS_DIR=${TOP_DIR}/expt_dirs # Set the path to the machine-specific test suite file. #----------------------------------------------------------------------- -auto_file=${scrfunc_dir}/machine_suites/${machine}.txt +auto_file=${scrfunc_dir}/machine_suites/${test_type}.${machine} if [ ! -f ${auto_file} ]; then - auto_file=${scrfunc_dir}/machine_suites/fundamental.txt + auto_file=${scrfunc_dir}/machine_suites/${test_type} fi #---------------------------------------------------------------------- # Use exec_subdir consistent with the automated build. #---------------------------------------------------------------------- -exec_subdir='install_intel/exec' +exec_subdir="install_${compiler}/exec" #----------------------------------------------------------------------- # Run E2E Tests @@ -84,7 +87,16 @@ echo "-- Load environment =>" $env_file source ${SRW_APP_DIR}/etc/lmod-setup.sh ${machine} module use ${env_path} module load ${env_file} -conda activate regional_workflow + +if [[ "${platform}" == 'cheyenne' ]]; then + conda activate "/glade/p/ral/jntp/UFS_SRW_app/conda/regional_workflow" +else + if [[ "${platform}" == 'noaacloud' && -z "${PROJ_LIB-}" ]]; then + PROJ_LIB='' + fi + + conda activate regional_workflow +fi module list @@ -94,6 +106,7 @@ module list machine=${machine} \ account=${account} \ exec_subdir=${exec_subdir} \ + compiler=${compiler} \ debug="TRUE" \ verbose="TRUE" \ run_envir="community"