Merge 'trilinos/Trilinos:develop' (e4a8b54) into 'tcad-charon/Trilino…

…s:develop' (4e16dc0). * trilinos-develop: (32 commits) Xpetra: Fix CMake warnings about missing source file extensions Tpetra unit test for issue trilinos#9583 (trilinos#9819) Testing on Geminga: Use new SEMS modules Stokhos: Include power_of_two from Kokkos Percept snapshot 12/20/21 Krino snapshot 12/20/21 STK: Snapshot 12-20-21 08:16 Tempus: Deprecate Some Functions Tpetra: Enable deprecated code warnings for capabilities changed during UVM-requirement removal (trilinos#10014) Tempus: Deprecate Some Functions Amesos2:SuperLU_MT : use current number of threads as the default "nprocs" parameter value Sacado: reneable dfad comm tests MueLu: Delete file that was commited by accident MueLu: Phase 2a - optionally match ML MueLu: Make Phase2a do the same as ML Tpetra: Adding Brian's feedback Tpetra: Modifying WDV to catch resizing one (but not both) child views a little easier Ctest: Because Jonathan can spell and I cannot Ctest: Disabling KokkosCore_UnitTest_CudaTimingBased on ascicgpu031 MueLu : put getDataNonConst in a separate scope to avoid "Cannot access data on device while a host view is alive" error ...
tcad-charon · Jan 4, 2022 · 22fb4e8 · 22fb4e8
2 parents 4e16dc0 + e4a8b54
commit 22fb4e8
Show file tree

Hide file tree

Showing 158 changed files with 4,619 additions and 4,937 deletions.
diff --git a/cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.crs.cmake b/cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.crs.cmake
@@ -88,6 +88,9 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Shards OFF CACHE BOOL "Shards does not build" FORCE)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
+  # Select test disables
+  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing")
+
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"
 

diff --git a/cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.sake.cmake b/cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.sake.cmake
@@ -88,6 +88,10 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Shards OFF CACHE BOOL "Shards does not build" FORCE)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
+  # Select test disables
+  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing")
+
+
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"
 

diff --git a/cmake/ctest/drivers/geminga/cron_driver.sh b/cmake/ctest/drivers/geminga/cron_driver.sh
@@ -35,17 +35,24 @@ export TDD_HTTPS_PROXY=$https_proxy
 
 . ~/.bashrc
 
+# Machine independent cron_driver:
+SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
+
+
+export export MODULEPATH=$SCRIPT_DIR:$MODULEPATH
+
 # If you update the list of modules, go to ~/code/trilinos-test/trilinos/ and
 # do "git pull". Otherwise, the tests could fail on the first night, as we
 # would first run old cron_driver.sh and only then pull
 
 # ===========================================================================
 export CTEST_CONFIGURATION="default"
-module load sems-cmake/3.17.1
-module load sems-gcc/5.3.0
-module load sems-openmpi/1.10.1
-module load sems-superlu/4.3/base
-module load sems-git/2.10.1
+# module load sems-cmake/3.17.1
+# module load sems-gcc/5.3.0
+# module load sems-openmpi/1.10.1
+# module load sems-superlu/4.3/base
+# module load sems-git/2.10.1
+module load muelu-gcc
 
 # Remove colors (-fdiagnostics-color) from OMPI flags
 # It may result in non-XML characters on the Dashboard
@@ -55,34 +62,32 @@ export OMPI_CXXFLAGS=`echo $OMPI_CXXFLAGS | sed 's/-fdiagnostics-color//'`
 echo "Configuration = $CTEST_CONFIGURATION"
 env
 
-export OMP_NUM_THREADS=2
+# export OMP_NUM_THREADS=2
 
-# Machine independent cron_driver:
-SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
 $SCRIPT_DIR/../cron_driver.py
 
-module unload sems-superlu/4.3/base
-module unload sems-openmpi/1.10.1
-module unload sems-gcc/5.3.0
-module unload sems-cmake/3.17.1
+# module unload sems-superlu/4.3/base
+# module unload sems-openmpi/1.10.1
+# module unload sems-gcc/5.3.0
+# module unload sems-cmake/3.17.1
 # ===========================================================================
 export CTEST_CONFIGURATION="nvcc_wrapper"
 #module load openmpi/1.10.0
 #module load gcc/4.9.2
 #module load cuda/7.5-gcc
 #module load nvcc-wrapper/gcc
 
-module load sems-env
-module load sems-cmake/3.17.1
-module load sems-gcc/8.3.0
-module load sems-boost/1.69.0/base
-module load sems-python/2.7.9
-module load sems-zlib/1.2.8/base
-module load sems-openmpi/4.0.2
-module load sems-cuda/10.1
-module load sems-cuda_openmpi/4.0.2/base
-module load sems-superlu/4.3
-module load sems-netcdf/4.7.3/parallel
+# module load sems-env
+# module load sems-cmake/3.17.1
+# module load sems-gcc/8.3.0
+# module load sems-boost/1.69.0/base
+# module load sems-python/2.7.9
+# module load sems-zlib/1.2.8/base
+# module load sems-openmpi/4.0.2
+# module load sems-cuda/10.1
+# module load sems-cuda_openmpi/4.0.2/base
+# module load sems-superlu/4.3
+# module load sems-netcdf/4.7.3/parallel
 # See Trilinos github issue #2115.
 export OMPI_CXX=/home/jhu/code/trilinos-test/trilinos/packages/kokkos/bin/nvcc_wrapper
 
@@ -97,29 +102,30 @@ echo "OMPI_CXXFLAGS after $OMPI_CXXFLAGS"
 echo "Configuration = $CTEST_CONFIGURATION"
 env
 
-export CUDA_LAUNCH_BLOCKING=1
-export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
+# export CUDA_LAUNCH_BLOCKING=1
+# export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
 # Only run on the Tesla K40, not the Quadro
-export CUDA_VISIBLE_DEVICES=0
+# export CUDA_VISIBLE_DEVICES=0
 # Machine independent cron_driver:
-SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
+# SCRIPT_DIR=`cd "\`dirname \"$0\"\`";pwd`
 $SCRIPT_DIR/../cron_driver.py
 
 #module unload nvcc-wrapper
 #module unload cuda
 #module unload gcc
 #module unload openmpi
-module unload sems-netcdf/4.7.3/parallel
-module unload sems-superlu/4.3
-module unload sems-cuda_openmpi/4.0.2/base
-module unload sems-cuda/10.1
-module unload sems-openmpi/4.0.2
-module unload sems-zlib/1.2.8/base
-module unload sems-python/2.7.9
-module unload sems-boost/1.69.0/base
-module unload sems-gcc/8.3.0
-module unload sems-cmake/3.17.1
-module unload sems-env
+# module unload sems-netcdf/4.7.3/parallel
+# module unload sems-superlu/4.3
+# module unload sems-cuda_openmpi/4.0.2/base
+# module unload sems-cuda/10.1
+# module unload sems-openmpi/4.0.2
+# module unload sems-zlib/1.2.8/base
+# module unload sems-python/2.7.9
+# module unload sems-boost/1.69.0/base
+# module unload sems-gcc/8.3.0
+# module unload sems-cmake/3.17.1
+# module unload sems-env
+module unload muelu-gcc
 # ===========================================================================
 
 echo

diff --git a/cmake/ctest/drivers/geminga/muelu-gcc.lua b/cmake/ctest/drivers/geminga/muelu-gcc.lua
@@ -0,0 +1,28 @@
+load("sems-gcc/8.3.0")
+load("sems-openmpi")
+load("sems-cmake")
+load("sems-ninja")
+load("sems-git")
+
+load("sems-superlu")
+
+load("sems-yaml-cpp")
+load("sems-hdf5")
+load("sems-netcdf-c")
+load("sems-parallel-netcdf")
+load("sems-zlib")
+
+load("sems-boost")
+load("sems-python")
+
+load("sems-metis")
+load("sems-parmetis")
+
+load("sems-cuda")
+
+pushenv("OMP_NUM_THREADS","2")
+pushenv("CUDA_LAUNCH_BLOCKING","1")
+pushenv("CUDA_MANAGED_FORCE_DEVICE_ALLOC","1")
+
+-- Only run on the Tesla K40, not the Quadro --
+pushenv("CUDA_VISIBLE_DEVICES","0")
diff --git a/packages/amesos2/src/Amesos2_Superlumt_def.hpp b/packages/amesos2/src/Amesos2_Superlumt_def.hpp
@@ -462,8 +462,8 @@ namespace Amesos2 {
 
     RCP<const Teuchos::ParameterList> valid_params = getValidParameters_impl();
 
-
-    data_.options.nprocs = parameterList->get<int>("nprocs", 1);
+    int default_nprocs = Kokkos::DefaultHostExecutionSpace::concurrency();
+    data_.options.nprocs = parameterList->get<int>("nprocs", default_nprocs);
 
     data_.options.trans = this->control_.useTranspose_ ? SLUMT::TRANS : SLUMT::NOTRANS;
     // SuperLU_MT "trans" option can override the Amesos2 option
@@ -521,10 +521,11 @@ namespace Amesos2 {
     if( is_null(valid_params) ){
       Teuchos::RCP<Teuchos::ParameterList> pl = Teuchos::parameterList();
 
+      int default_nprocs = Kokkos::DefaultHostExecutionSpace::concurrency();
       Teuchos::RCP<EnhancedNumberValidator<int> > nprocs_validator
         = Teuchos::rcp( new EnhancedNumberValidator<int>() );
-      nprocs_validator->setMin(1);
-      pl->set("nprocs", 1, "The number of processors to factorize with", nprocs_validator);
+      nprocs_validator->setMin(default_nprocs);
+      pl->set("nprocs", default_nprocs, "The number of processors to factorize with", nprocs_validator);
 
       setStringToIntegralParameter<SLUMT::trans_t>("trans", "NOTRANS",
                                                    "Solve for the transpose system or not",

diff --git a/packages/ifpack2/src/Ifpack2_RILUK_def.hpp b/packages/ifpack2/src/Ifpack2_RILUK_def.hpp
@@ -941,16 +941,21 @@ void RILUK<MatrixType>::compute ()
 
     using row_map_type = typename crs_matrix_type::local_matrix_device_type::row_map_type;
 
-    row_map_type L_rowmap  = L_->getLocalMatrixDevice().graph.row_map;
-    auto L_entries = L_->getLocalMatrixDevice().graph.entries;
-    auto L_values  = L_->getLocalValuesView();
-    row_map_type U_rowmap  = U_->getLocalMatrixDevice().graph.row_map;
-    auto U_entries = U_->getLocalMatrixDevice().graph.entries;
-    auto U_values  = U_->getLocalValuesView();
-
-    KokkosSparse::Experimental::spiluk_numeric( KernelHandle_.getRawPtr(), LevelOfFill_, 
-                                                A_local_rowmap_, A_local_entries_, A_local_values_, 
-                                                L_rowmap, L_entries, L_values, U_rowmap, U_entries, U_values );
+    {
+      auto lclL = L_->getLocalMatrixDevice();
+      row_map_type L_rowmap  = lclL.graph.row_map;
+      auto L_entries = lclL.graph.entries;
+      auto L_values  = L_->getLocalValuesView();
+
+      auto lclU = U_->getLocalMatrixDevice();
+      row_map_type U_rowmap  = lclU.graph.row_map;
+      auto U_entries = lclU.graph.entries;
+      auto U_values  = U_->getLocalValuesView();
+
+      KokkosSparse::Experimental::spiluk_numeric( KernelHandle_.getRawPtr(), LevelOfFill_, 
+                                                  A_local_rowmap_, A_local_entries_, A_local_values_, 
+                                                  L_rowmap, L_entries, L_values, U_rowmap, U_entries, U_values );
+    }
 
     L_->fillComplete (L_->getColMap (), A_local_->getRangeMap ());
     U_->fillComplete (A_local_->getDomainMap (), U_->getRowMap ());

diff --git a/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp b/packages/ifpack2/test/unit_tests/Ifpack2_UnitTestChebyshev.cpp
@@ -167,14 +167,14 @@ TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL(Ifpack2Chebyshev, Test0, Scalar, LocalOrdinal,
   prec.setParameters(params);
   prec.compute();
 
-  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,2e-2);
+  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,4.5e-2);
 
   params.set("eigen-analysis: type", "cg");
   params.set("chebyshev: eigenvalue max iterations",10);
   prec.setParameters(params);
   prec.compute();
 
-  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,2e-2);
+  TEST_FLOATING_EQUALITY(prec.getLambdaMaxForApply(),expectedLambdaMax,4.5e-2);
 }
 
 #define UNIT_TEST_GROUP_SC_LO_GO(Scalar,LocalOrdinal,GlobalOrdinal) \

diff --git a/packages/krino/cmake_install_test/build_Trilinos_krino_package.sh b/packages/krino/cmake_install_test/build_Trilinos_krino_package.sh
@@ -8,7 +8,15 @@ fi
 
 function execute() {
   stdbuf -o0 -e0 echo "% $@" ;
-  eval "$@" ;
+  if [ $# -gt 0 ] ; then
+    if [ $1 == "module" ] ; then
+      module "${@:2}" ;
+    elif [ $1 == "source" ] ; then
+      source "${@:2}" ;
+    else
+      eval "$@" ;
+    fi
+  fi
   if [ $? -ne 0 ] ; then
     echo "'$@' failed.";
     exit 1;
@@ -47,12 +55,8 @@ function pull_and_build_yaml()
 
     cd_to_new_dir ${productName}_build
 
-    export CC=gcc
-    export CXX=g++
     execute cmake -DCMAKE_BUILD_TYPE=${build_type^^} -DYAML_CPP_BUILD_TESTS=false -DCMAKE_INSTALL_PREFIX=../${productName}_install ../yaml-cpp
     make_and_install $productName
-    unset CC
-    unset CXX
 }
 
 function pull_Trilinos()
@@ -70,7 +74,8 @@ function pull_Trilinos()
     else
       execute cd Trilinos
       execute git checkout develop
-      execute git reset --hard origin/develop
+      execute git clean -fd
+      execute git reset --hard
       execute git pull
       execute cd ..
     fi
@@ -94,20 +99,25 @@ function build_krino_Trilinos_package()
 
 function setup_environment()
 {
-    module purge
+    execute source /etc/profile.d/modules.sh
+    execute module purge
 
     # compiler environment
-    module load cde/dev/cmake/3.19.2
-    module load cde/dev/compiler/gcc/7.2.0
-    module load cde/dev/gcc/7.2.0/openmpi/4.0.5
+    export MODULEPATH=/projects/cde/modules:$MODULEPATH
+    execute module load cde/v2/cmake/3.19.2
+    execute module load cde/v2/compiler/gcc/7.2.0
+    execute module load cde/v2/gcc/7.2.0/openmpi/4.0.5
 
     #TPLs
-    module load cde/dev/gcc/7.2.0/netlib-lapack/3.8.0
-    module load cde/dev/gcc/7.2.0/hdf5/1.10.6
-    module load cde/dev/gcc/7.2.0/netcdf-c/4.7.3
-    module load cde/dev/gcc/7.2.0/parallel-netcdf/1.12.1
-    module load cde/dev/gcc/7.2.0/metis/5.1.0
-    module load cde/dev/gcc/7.2.0/parmetis/4.0.3
+    execute module load cde/v2/gcc/7.2.0/netlib-lapack/3.8.0
+    execute module load cde/v2/gcc/7.2.0/hdf5/1.10.6
+    execute module load cde/v2/gcc/7.2.0/netcdf-c/4.7.3
+    execute module load cde/v2/gcc/7.2.0/parallel-netcdf/1.12.1
+    execute module load cde/v2/gcc/7.2.0/metis/5.1.0
+    execute module load cde/v2/gcc/7.2.0/parmetis/4.0.3
+
+    execute module list
+    execute env
 }
 
 output_dir=$1

diff --git a/packages/krino/cmake_install_test/build_sierra_krino_as_Trilinos_package.sh b/packages/krino/cmake_install_test/build_sierra_krino_as_Trilinos_package.sh
@@ -2,7 +2,15 @@
 
 function execute() {
   stdbuf -o0 -e0 echo "% $@" ;
-  eval "$@" ;
+  if [ $# -gt 0 ] ; then
+    if [ $1 == "module" ] ; then
+      module "${@:2}" ;
+    elif [ $1 == "source" ] ; then
+      source "${@:2}" ;
+    else
+      eval "$@" ;
+    fi
+  fi
   if [ $? -ne 0 ] ; then
     echo "'$@' failed.";
     exit 1;
@@ -36,12 +44,8 @@ function build_yaml()
     execute tar -xzf ${sierra_proj}/TPLs_src/spack/spack_tpls/yaml-cpp/*.tar.gz
     cd_to_new_dir ${productName}_build
 
-    export CC=gcc
-    export CXX=g++
     execute cmake -DCMAKE_BUILD_TYPE=${build_type^^} -DYAML_CPP_BUILD_TESTS=false -DCMAKE_INSTALL_PREFIX=../${productName}_install ../yaml-cpp
     make_and_install $productName
-    unset CC
-    unset CXX
 }
 
 function setup_trilinos_with_krino()
@@ -59,7 +63,8 @@ function setup_trilinos_with_krino()
     else
       execute cd Trilinos
       execute git checkout develop
-      execute git reset --hard origin/develop
+      execute git clean -fd
+      execute git reset --hard
       execute git pull
       execute cd ..
     fi
@@ -89,11 +94,12 @@ function build_trilinos_with_krino()
 
 function setup_environment()
 {
-    source ${output_dir}/trilinos/Trilinos/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh
+    execute source /etc/profile.d/modules.sh
+    execute source ${output_dir}/trilinos/Trilinos/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh
 
     # fixup for python 2
-    module unload sems-python/3.5.2
-    module load sems-python/2.7.9 
+    execute module unload sems-python/3.5.2
+    execute module load sems-python/2.7.9 
 }
 
 function runTests()

diff --git a/packages/krino/cmake_install_test/load_gcc_modules b/packages/krino/cmake_install_test/load_gcc_modules