A few more tweaks to the release scripts.

Background:

We are having a lot of trouble the the Cray environment during this release
cycle.  Unloading modules is causing errors and aborting builds.  I think I have
worked through all of the issues and updated the 'release_cray.sh' script.

Also - the new python scripts in Jayenne require python 2.7+.  The default
python on Toss2 (pinto, wolf) is 2.6 and so my initial release on these machines
was defective.  I've updated the build/run environments for toss machines to
load python/3.6.

Changes:

+ Cray machines:
  + Require the build environment on Crays to be 'clean' when the release script
    is started.  This is done by aborting if the environment variable
    ParMETIS_ROOT_DIR is defined.
  + Ensure that the 'craype-mic-knl' module is not loaded when attempting to
    unload other Cray PrgEnv modules.
  + Ensure that our TPL modules are unloaded before attempting to unload Cray
  PrgEnv modules.
+ Toss2 machines:
  + Ensure that the python/3.6 module is loaded to properly support Jayenne.
+ If a test fails, run it a second time.  This seems ridiculous, but there are
  so many system issues right now that we need this information.

regression/scripts/common.sh

@@ -508,7 +508,11 @@ function install_versions
         run "ctest -L nr -j $test_pe" ;;
       *)
         # run all tests
-        run "ctest -j $test_pe --output-on-failure" ;;
+        run "ctest -j $test_pe --output-on-failure"
+        if [[ $? != 0 ]]; then
+          run "ctest -j $test_pe --output-on-failure --rerun-failed"
+        fi
+        ;;
     esac
   fi
   if ! test ${build_permissions:-notset} = "notset"; then

regression/scripts/release_cray.sh

@@ -36,9 +36,17 @@ case $target in
   t[rt]-fe* | t[rt]-login* )
     environments="intel18env intel18env-knl intel17env intel17env-knl" ;;
 esac
-
+export VENDOR_DIR=/usr/projects/draco/vendors
+if [[ -d $ParMETIS_ROOT_DIR ]]; then
+  echo "ERROR: This script should be run from a clean environment."
+  echo "       Try running 'rmdracoenv'."
+  exit 1
+fi
 function intel18env()
 {
+if [[ ${CRAY_CPU_TARGET} == mic-knl ]]; then
+  run "module swap craype-mic-knl craype-haswell"
+fi
 run "module load user_contrib friendly-testing"
 run "module unload cmake numdiff git"
 run "module unload gsl random123 eospac"
@@ -70,14 +78,16 @@ export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
 
 function intel18env-knl()
 {
+if [[ ${CRAY_CPU_TARGET} == mic-knl ]]; then
+  run "module swap craype-mic-knl craype-haswell"
+fi
 run "module load user_contrib friendly-testing"
 run "module unload cmake numdiff git"
 run "module unload gsl random123 eospac"
 run "module unload trilinos ndi"
 run "module unload superlu-dist metis parmetis"
+run "module unload csk lapack"
 run "module unload PrgEnv-intel PrgEnv-pgi PrgEnv-cray PrgEnv-gnu"
-run "module unload csk"
-run "module unload lapack intel"
 run "module unload intel gcc"
 run "module unload papi perftools"
 run "module load PrgEnv-intel"
@@ -103,6 +113,9 @@ export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
 
 function intel17env()
 {
+if [[ ${CRAY_CPU_TARGET} == mic-knl ]]; then
+  run "module swap craype-mic-knl craype-haswell"
+fi
 run "module load user_contrib friendly-testing"
 run "module unload cmake numdiff git"
 run "module unload gsl random123 eospac"
@@ -134,14 +147,16 @@ export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
 
 function intel17env-knl()
 {
+if [[ ${CRAY_CPU_TARGET} == mic-knl ]]; then
+  run "module swap craype-mic-knl craype-haswell"
+fi
 run "module load user_contrib friendly-testing"
 run "module unload cmake numdiff git"
 run "module unload gsl random123 eospac"
 run "module unload trilinos ndi"
 run "module unload superlu-dist metis parmetis"
+run "module unload csk lapack"
 run "module unload PrgEnv-intel PrgEnv-pgi PrgEnv-cray PrgEnv-gnu"
-run "module unload csk"
-run "module unload lapack intel"
 run "module unload intel gcc"
 run "module unload papi perftools"
 run "module load PrgEnv-intel"

regression/scripts/release_toss.sh

@@ -38,7 +38,7 @@ function intel1802env()
   run "module purge"
   run "module use --append ${VENDOR_DIR}-ec/modulefiles"
   run "module load friendly-testing user_contrib"
-  run "module load cmake git numdiff"
+  run "module load cmake git numdiff python/3.6-anaconda-5.0.1"
   run "module load intel/18.0.2 openmpi/2.1.2"
   run "module load random123 eospac/6.3.0 gsl"
   run "module load mkl metis ndi csk"
@@ -51,7 +51,7 @@ function intel1704env()
   run "module purge"
   run "module use --append ${VENDOR_DIR}-ec/modulefiles"
   run "module load friendly-testing user_contrib"
-  run "module load cmake git numdiff"
+  run "module load cmake git numdiff python/3.6-anaconda-5.0.1"
   run "module load intel/17.0.4 openmpi/2.1.2"
   run "module load random123 eospac/6.3.0 gsl"
   run "module load mkl metis ndi csk"
@@ -64,7 +64,7 @@ function gcc640env()
   run "module purge"
   run "module use --append ${VENDOR_DIR}-ec/modulefiles"
   run "module load friendly-testing user_contrib"
-  run "module load cmake git numdiff"
+  run "module load cmake git numdiff python/3.6-anaconda-5.0.1"
   run "module load gcc/6.4.0 openmpi/2.1.2"
   run "module load random123 eospac/6.3.0 gsl"
   run "module load mkl metis ndi"