Merge 'trilinos/Trilinos:develop' (7b1fc5a) into 'tcad-charon/Trilino…

…s:develop' (4e16dc0).

* trilinos-develop: (24 commits)
  Tpetra unit test for issue trilinos#9583 (trilinos#9819)
  Testing on Geminga: Use new SEMS modules
  Tpetra:  Enable deprecated code warnings for capabilities changed during UVM-requirement removal (trilinos#10014)
  Sacado: reneable dfad comm tests
  MueLu: Delete file that was commited by accident
  MueLu: Phase 2a - optionally match ML
  MueLu: Make Phase2a do the same as ML
  Tpetra: Adding Brian's feedback
  Tpetra: Modifying WDV to catch resizing one (but not both) child views a little easier
  Ctest: Because Jonathan can spell and I cannot
  Ctest: Disabling KokkosCore_UnitTest_CudaTimingBased on ascicgpu031
  MueLu : put getDataNonConst in a separate scope to avoid "Cannot access data on device while a host view is alive" error
  Ifpack2: Bump up tolerance in Chebyshev unit test
  Phalanx: fix cuda issue due to changes for hip support
  Tpetra: Fixing UVM-free issues
  Tpetra: Brian's feedback
  Tpetra: Brian's feedback
  Phalanx: add support for HIP
  Tpetra:  Fixing subtle UVM issue
  Tpetra:  Fixing subtle UVM issue
  ...

cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.crs.cmake

@@ -88,6 +88,9 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Shards OFF CACHE BOOL "Shards does not build" FORCE)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
+  # Select test disables
+  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing")
+
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"
 

cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.sake.cmake

@@ -88,6 +88,10 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Shards OFF CACHE BOOL "Shards does not build" FORCE)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
+  # Select test disables
+  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing")
+
+
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"
 

cmake/ctest/drivers/geminga/cron_driver.sh

@@ -35,17 +35,24 @@ export TDD_HTTPS_PROXY=$https_proxy
 
 . ~/.bashrc
 
+# Machine independent cron_driver:
+SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
+
+
+export export MODULEPATH=$SCRIPT_DIR:$MODULEPATH
+
 # If you update the list of modules, go to ~/code/trilinos-test/trilinos/ and
 # do "git pull". Otherwise, the tests could fail on the first night, as we
 # would first run old cron_driver.sh and only then pull
 
 # ===========================================================================
 export CTEST_CONFIGURATION="default"
-module load sems-cmake/3.17.1
-module load sems-gcc/5.3.0
-module load sems-openmpi/1.10.1
-module load sems-superlu/4.3/base
-module load sems-git/2.10.1
+# module load sems-cmake/3.17.1
+# module load sems-gcc/5.3.0
+# module load sems-openmpi/1.10.1
+# module load sems-superlu/4.3/base
+# module load sems-git/2.10.1
+module load muelu-gcc
 
 # Remove colors (-fdiagnostics-color) from OMPI flags
 # It may result in non-XML characters on the Dashboard
@@ -55,34 +62,32 @@ export OMPI_CXXFLAGS=`echo $OMPI_CXXFLAGS | sed 's/-fdiagnostics-color//'`
 echo "Configuration = $CTEST_CONFIGURATION"
 env
 
-export OMP_NUM_THREADS=2
+# export OMP_NUM_THREADS=2
 
-# Machine independent cron_driver:
-SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
 $SCRIPT_DIR/../cron_driver.py
 
-module unload sems-superlu/4.3/base
-module unload sems-openmpi/1.10.1
-module unload sems-gcc/5.3.0
-module unload sems-cmake/3.17.1
+# module unload sems-superlu/4.3/base
+# module unload sems-openmpi/1.10.1
+# module unload sems-gcc/5.3.0
+# module unload sems-cmake/3.17.1
 # ===========================================================================
 export CTEST_CONFIGURATION="nvcc_wrapper"
 #module load openmpi/1.10.0
 #module load gcc/4.9.2
 #module load cuda/7.5-gcc
 #module load nvcc-wrapper/gcc
 
-module load sems-env
-module load sems-cmake/3.17.1
-module load sems-gcc/8.3.0
-module load sems-boost/1.69.0/base
-module load sems-python/2.7.9
-module load sems-zlib/1.2.8/base
-module load sems-openmpi/4.0.2
-module load sems-cuda/10.1
-module load sems-cuda_openmpi/4.0.2/base
-module load sems-superlu/4.3
-module load sems-netcdf/4.7.3/parallel
+# module load sems-env
+# module load sems-cmake/3.17.1
+# module load sems-gcc/8.3.0
+# module load sems-boost/1.69.0/base
+# module load sems-python/2.7.9
+# module load sems-zlib/1.2.8/base
+# module load sems-openmpi/4.0.2
+# module load sems-cuda/10.1
+# module load sems-cuda_openmpi/4.0.2/base
+# module load sems-superlu/4.3
+# module load sems-netcdf/4.7.3/parallel
 # See Trilinos github issue #2115.
 export OMPI_CXX=/home/jhu/code/trilinos-test/trilinos/packages/kokkos/bin/nvcc_wrapper
 
@@ -97,29 +102,30 @@ echo "OMPI_CXXFLAGS after $OMPI_CXXFLAGS"
 echo "Configuration = $CTEST_CONFIGURATION"
 env
 
-export CUDA_LAUNCH_BLOCKING=1
-export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
+# export CUDA_LAUNCH_BLOCKING=1
+# export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
 # Only run on the Tesla K40, not the Quadro
-export CUDA_VISIBLE_DEVICES=0
+# export CUDA_VISIBLE_DEVICES=0
 # Machine independent cron_driver:
-SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
+# SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
 $SCRIPT_DIR/../cron_driver.py
 
 #module unload nvcc-wrapper
 #module unload cuda
 #module unload gcc
 #module unload openmpi
-module unload sems-netcdf/4.7.3/parallel
-module unload sems-superlu/4.3
-module unload sems-cuda_openmpi/4.0.2/base
-module unload sems-cuda/10.1
-module unload sems-openmpi/4.0.2
-module unload sems-zlib/1.2.8/base
-module unload sems-python/2.7.9
-module unload sems-boost/1.69.0/base
-module unload sems-gcc/8.3.0
-module unload sems-cmake/3.17.1
-module unload sems-env
+# module unload sems-netcdf/4.7.3/parallel
+# module unload sems-superlu/4.3
+# module unload sems-cuda_openmpi/4.0.2/base
+# module unload sems-cuda/10.1
+# module unload sems-openmpi/4.0.2
+# module unload sems-zlib/1.2.8/base
+# module unload sems-python/2.7.9
+# module unload sems-boost/1.69.0/base
+# module unload sems-gcc/8.3.0
+# module unload sems-cmake/3.17.1
+# module unload sems-env
+module unload muelu-gcc
 # ===========================================================================
 
 echo

cmake/ctest/drivers/geminga/muelu-gcc.lua

@@ -0,0 +1,28 @@
+load("sems-gcc/8.3.0")
+load("sems-openmpi")
+load("sems-cmake")
+load("sems-ninja")
+load("sems-git")
+
+load("sems-superlu")
+
+load("sems-yaml-cpp")
+load("sems-hdf5")
+load("sems-netcdf-c")
+load("sems-parallel-netcdf")
+load("sems-zlib")
+
+load("sems-boost")
+load("sems-python")
+
+load("sems-metis")
+load("sems-parmetis")
+
+load("sems-cuda")
+
+pushenv("OMP_NUM_THREADS","2")
+pushenv("CUDA_LAUNCH_BLOCKING","1")
+pushenv("CUDA_MANAGED_FORCE_DEVICE_ALLOC","1")
+
+-- Only run on the Tesla K40, not the Quadro --
+pushenv("CUDA_VISIBLE_DEVICES","0")

packages/ifpack2/src/Ifpack2_RILUK_def.hpp

@@ -941,16 +941,21 @@ void RILUK<MatrixType>::compute ()
 
     using row_map_type = typename crs_matrix_type::local_matrix_device_type::row_map_type;
 
-    row_map_type L_rowmap  = L_->getLocalMatrixDevice().graph.row_map;
-    auto L_entries = L_->getLocalMatrixDevice().graph.entries;
-    auto L_values  = L_->getLocalValuesView();
-    row_map_type U_rowmap  = U_->getLocalMatrixDevice().graph.row_map;
-    auto U_entries = U_->getLocalMatrixDevice().graph.entries;
-    auto U_values  = U_->getLocalValuesView();
-
-    KokkosSparse::Experimental::spiluk_numeric( KernelHandle_.getRawPtr(), LevelOfFill_, 
-                                                A_local_rowmap_, A_local_entries_, A_local_values_, 
-                                                L_rowmap, L_entries, L_values, U_rowmap, U_entries, U_values );
+    {
+      auto lclL = L_->getLocalMatrixDevice();
+      row_map_type L_rowmap  = lclL.graph.row_map;
+      auto L_entries = lclL.graph.entries;
+      auto L_values  = L_->getLocalValuesView();
+
+      auto lclU = U_->getLocalMatrixDevice();
+      row_map_type U_rowmap  = lclU.graph.row_map;
+      auto U_entries = lclU.graph.entries;
+      auto U_values  = U_->getLocalValuesView();
+
+      KokkosSparse::Experimental::spiluk_numeric( KernelHandle_.getRawPtr(), LevelOfFill_, 
+                                                  A_local_rowmap_, A_local_entries_, A_local_values_, 
+                                                  L_rowmap, L_entries, L_values, U_rowmap, U_entries, U_values );
+    }
 
     L_->fillComplete (L_->getColMap (), A_local_->getRangeMap ());
     U_->fillComplete (A_local_->getDomainMap (), U_->getRowMap ());

packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp

@@ -167,14 +167,14 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2Chebyshev, Test0, Scalar, LocalOrdinal,
   prec.setParameters(params);
   prec.compute();
 
-  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,2e-2);
+  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,4.5e-2);
 
   params.set("eigen-analysis: type", "cg");
   params.set("chebyshev: eigenvalue max iterations",10);
   prec.setParameters(params);
   prec.compute();
 
-  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,2e-2);
+  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,4.5e-2);
 }
 
 #define UNIT_TEST_GROUP_SC_LO_GO(Scalar,LocalOrdinal,GlobalOrdinal) \