trilinos · trilinos-autotester · Jun 10, 2021 · Jun 10, 2021 · Jun 10, 2021 · Jul 8, 2021
diff --git a/PackagesList.cmake b/PackagesList.cmake
@@ -112,6 +112,7 @@ TRIBITS_REPOSITORY_DEFINE_PACKAGES(
   Compadre              packages/compadre                 ST
   STK                   packages/stk                      PT # Depends on boost
   Percept               packages/percept                  PT # Depends on boost
+  Krino                 packages/krino                    PT # Depends on boost
   SCORECapf_zoltan      SCOREC/zoltan                     ST
   SCORECapf_stk         SCOREC/stk                        ST
   SCORECma              SCOREC/ma                         ST

diff --git a/cmake/std/PullRequestLinuxCuda10.1.105TestingSettings.cmake b/cmake/std/PullRequestLinuxCuda10.1.105TestingSettings.cmake
@@ -78,7 +78,9 @@ set (TPL_DLlib_LIBRARIES "-ldl" CACHE FILEPATH "Set by default for CUDA PR testi
 # The compile times for two Panzer files went up to over 6 hours. This
 # turns off one feature that allows these in about 24 minutes. Please remove
 # when issue #7532 is resolved.
-set (Sacado_NEW_FAD_DESIGN_IS_DEFAULT OFF CACHE BOOL "Temporary fix for issue #7532" )
+# Compile time issues addressed by #8377. Commenting out the override of 
+# Sacado_NEW_FAD_DESIGN_IS_DEFAULT to return to default settings.
+#set (Sacado_NEW_FAD_DESIGN_IS_DEFAULT OFF CACHE BOOL "Temporary fix for issue #7532" )
 
 # Disable some packages that can't be tested with this PR build
 set (Trilinos_ENABLE_ShyLU_NodeTacho OFF CACHE BOOL

diff --git a/cmake/std/PullRequestLinuxCuda10.1.105uvmOffTestingSettings.cmake b/cmake/std/PullRequestLinuxCuda10.1.105uvmOffTestingSettings.cmake
@@ -77,7 +77,9 @@ set (TPL_DLlib_LIBRARIES "-ldl" CACHE FILEPATH "Set by default for CUDA PR testi
 # The compile times for two Panzer files went up to over 6 hours. This
 # turns off one feature that allows these in about 24 minutes. Please remove
 # when issue #7532 is resolved.
-set (Sacado_NEW_FAD_DESIGN_IS_DEFAULT OFF CACHE BOOL "Temporary fix for issue #7532" )
+# Compile time issues addressed by #8377. Commenting out the override of 
+# Sacado_NEW_FAD_DESIGN_IS_DEFAULT to return to default settings.
+#set (Sacado_NEW_FAD_DESIGN_IS_DEFAULT OFF CACHE BOOL "Temporary fix for issue #7532" )
 
 # Disable some packages that can't be tested with this PR build
 set (Trilinos_ENABLE_ShyLU_NodeTacho OFF CACHE BOOL
@@ -146,7 +148,6 @@ set (Domi_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
 set (Kokkos_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
 set (KokkosKernels_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
 set (ROL_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
-set (Sacado_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
 set (SEACAS_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
 set (ShyLU_DD_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")
 set (STK_ENABLE_TESTS OFF CACHE BOOL "Turn off tests for non-UVM build")

diff --git a/cmake/std/PullRequestLinuxCuda10.1.243TestingSettings.cmake b/cmake/std/PullRequestLinuxCuda10.1.243TestingSettings.cmake
@@ -84,7 +84,9 @@ set (TPL_DLlib_LIBRARIES "-ldl" CACHE FILEPATH "Set by default for CUDA PR testi
 # The compile times for two Panzer files went up to over 6 hours. This
 # turns off one feature that allows these in about 24 minutes. Please remove
 # when issue #7532 is resolved.
-set (Sacado_NEW_FAD_DESIGN_IS_DEFAULT OFF CACHE BOOL "Temporary fix for issue #7532" )
+# Compile time issues addressed by #8377. Commenting out the override of 
+# Sacado_NEW_FAD_DESIGN_IS_DEFAULT to return to default settings.
+#set (Sacado_NEW_FAD_DESIGN_IS_DEFAULT OFF CACHE BOOL "Temporary fix for issue #7532" )
 
 # Disable some packages that can't be tested with this PR build
 set (Trilinos_ENABLE_ShyLU_NodeTacho OFF CACHE BOOL

diff --git a/cmake/std/atdm/README.md b/cmake/std/atdm/README.md
@@ -1572,7 +1572,7 @@ each system</a>.
 These contributed configurations are used just like any other custom
 configuration as described in <a
 href="#custom-systems-and-configurations">Custom systems and
-configurations</a>..  For example, to load the contributed custom 'weaver'
+configurations</a>.  For example, to load the contributed custom 'weaver'
 configuration to do a CUDA optimized build, do:
 
 ```

diff --git a/cmake/std/atdm/contributed/weaver/environment.sh b/cmake/std/atdm/contributed/weaver/environment.sh
@@ -177,7 +177,7 @@ elif [[ "$ATDM_CONFIG_COMPILER" == "CUDA"* ]] ; then
 fi
 
 # Ninja
-module load ninja/1.7.2
+#module load ninja/1.7.2
 
 # CMake
 #module swap cmake/3.6.2 cmake/3.12.3

diff --git a/packages/adelus/CMakeLists.txt b/packages/adelus/CMakeLists.txt
@@ -55,7 +55,7 @@ TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_TIMING
 TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_CUDAHOSTPINNED
   CUDA_HOST_PINNED_MPI
   "Use Cuda Host Pinned memory for MPI."
-  OFF )
+  ON )
 
 TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_USEDEEPCOPY
   USE_DEEPCOPY

diff --git a/packages/adelus/src/Adelus_perm1.hpp b/packages/adelus/src/Adelus_perm1.hpp
@@ -300,6 +300,9 @@ namespace Adelus {
     int ptr1_idx, myfirstrow;
 
   #ifdef GET_TIMING
+  #if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+    double t1, copyhostpinnedtime;
+  #endif
     double t2;
     double totalpermtime;
   #endif
@@ -316,10 +319,18 @@ namespace Adelus {
   #endif
     typedef typename ZDView::device_type::memory_space memory_space;
     typedef Kokkos::View<value_type**, Kokkos::LayoutLeft, memory_space> ViewMatrixType;
-
+
+  #if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+    typedef Kokkos::View<value_type**, Kokkos::LayoutLeft, Kokkos::CudaHostPinnedSpace> View2DHostPinnType;//CudaHostPinnedSpace
+  #endif
+
     if (my_rhs_ > 0) {
 
-      ViewMatrixType rhs_temp ( "rhs_temp", nrows_matrix, my_rhs_ );//allocate full-size RHS vectors 
+      ViewMatrixType rhs_temp ( "rhs_temp", nrows_matrix, my_rhs_ );//allocate full-size RHS vectors
+  #if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+      View2DHostPinnType h_rhs_temp( "h_rhs_temp", nrows_matrix, my_rhs_ );
+  #endif
+
       Kokkos::deep_copy(rhs_temp, 0);//initialize with 0s
 
       ncols_proc1 = ncols_matrix/nprocs_row;
@@ -357,18 +368,35 @@ namespace Adelus {
 
         ptr1_idx++;
       }
+  #if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+  #ifdef GET_TIMING
+      t1 = MPI_Wtime();
+  #endif
+      Kokkos::deep_copy(h_rhs_temp,rhs_temp);
+  #ifdef GET_TIMING
+      copyhostpinnedtime = (MPI_Wtime()-t1);
+  #endif
 
+      MPI_Allreduce( MPI_IN_PLACE, h_rhs_temp.data(), nrows_matrix*my_rhs_, ADELUS_MPI_DATA_TYPE, MPI_SUM, col_comm);
+
+      Kokkos::deep_copy( subview(ZV, Kokkos::ALL(), Kokkos::make_pair(0, my_rhs_)), 
+                         subview(h_rhs_temp, Kokkos::make_pair(myfirstrow-1, myfirstrow-1+my_rows), Kokkos::ALL()) );
+  #else //CUDA-aware MPI
       MPI_Allreduce( MPI_IN_PLACE, rhs_temp.data(), nrows_matrix*my_rhs_, ADELUS_MPI_DATA_TYPE, MPI_SUM, col_comm);
 
       Kokkos::deep_copy( subview(ZV, Kokkos::ALL(), Kokkos::make_pair(0, my_rhs_)), 
                          subview(rhs_temp, Kokkos::make_pair(myfirstrow-1, myfirstrow-1+my_rows), Kokkos::ALL()) );
-
+  #endif  
     }
 
   #ifdef GET_TIMING
     totalpermtime = MPI_Wtime() - t2;
   #endif
+
   #ifdef GET_TIMING
+  #if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+    showtime("Time to copy dev mem --> host pinned mem",&copyhostpinnedtime);   
+  #endif
     showtime("Total time in perm",&totalpermtime);
   #endif
   }

diff --git a/packages/adelus/src/Adelus_solve.hpp b/packages/adelus/src/Adelus_solve.hpp
@@ -149,9 +149,9 @@ void back_solve6(ZDView& ZV)
   double t1,t2;
   double allocviewtime,eliminaterhstime,bcastrowtime,updrhstime,xchgrhstime;
   double totalsolvetime;
-//#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
-//  double copyhostpinnedtime;
-//#endif
+#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+  double copyhostpinnedtime;
+#endif
 #endif
 
   MPI_Request msgrequest;
@@ -181,9 +181,9 @@ void back_solve6(ZDView& ZV)
 
 #ifdef GET_TIMING
   allocviewtime=eliminaterhstime=bcastrowtime=updrhstime=xchgrhstime=0.0;
-//#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
-//  copyhostpinnedtime=0.0;
-//#endif
+#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+  copyhostpinnedtime=0.0;
+#endif
 
   t1 = MPI_Wtime();
 #endif
@@ -251,15 +251,15 @@ void back_solve6(ZDView& ZV)
 #endif
         }
 
-//#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
-//#ifdef GET_TIMING
-//        t1 = MPI_Wtime();
-//#endif
-//        Kokkos::deep_copy(h_row1,row1);
-//#ifdef GET_TIMING
-//        copyhostpinnedtime += (MPI_Wtime()-t1);
-//#endif
-//#endif
+#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+#ifdef GET_TIMING
+        t1 = MPI_Wtime();
+#endif
+        Kokkos::deep_copy(h_row1,row1);
+#ifdef GET_TIMING
+        copyhostpinnedtime += (MPI_Wtime()-t1);
+#endif
+#endif
 
 #ifdef GET_TIMING
         t1 = MPI_Wtime();
@@ -268,27 +268,27 @@ void back_solve6(ZDView& ZV)
         type[0]  = SOCOLTYPE+j;
 
         //MPI_Bcast((char *) row1, bytes[0], MPI_CHAR, mesh_row(root), col_comm);
-//#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
-//        MPI_Bcast(reinterpret_cast<char *>(h_row1.data()), bytes[0], MPI_CHAR, mesh_row(root), col_comm);
-//#else //CUDA-aware MPI -- Note: Looks like MPI_Bcast is still working well with device (cuda) pointers (and faster than using cuda host pinned memory)
+#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+        MPI_Bcast(reinterpret_cast<char *>(h_row1.data()), bytes[0], MPI_CHAR, mesh_row(root), col_comm);
+#else //CUDA-aware MPI
         MPI_Bcast(reinterpret_cast<char *>(row1.data()), bytes[0], MPI_CHAR, mesh_row(root), col_comm);		
-//#endif
+#endif
         // added this barrier for CPLANT operation
 
         MPI_Barrier(col_comm);
 #ifdef GET_TIMING
         bcastrowtime += (MPI_Wtime()-t1);
 #endif
 
-//#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
-//#ifdef GET_TIMING
-//        t1 = MPI_Wtime();
-//#endif
-//        Kokkos::deep_copy(row1,h_row1);
-//#ifdef GET_TIMING
-//        copyhostpinnedtime += (MPI_Wtime()-t1);
-//#endif
-//#endif
+#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+#ifdef GET_TIMING
+        t1 = MPI_Wtime();
+#endif
+        Kokkos::deep_copy(row1,h_row1);
+#ifdef GET_TIMING
+        copyhostpinnedtime += (MPI_Wtime()-t1);
+#endif
+#endif
 
 #ifdef GET_TIMING
         t1 = MPI_Wtime();
@@ -376,9 +376,9 @@ void back_solve6(ZDView& ZV)
   showtime("Time to eliminate rhs",&eliminaterhstime);
   showtime("Time to bcast temp row",&bcastrowtime);
   showtime("Time to update rhs",&updrhstime);
-//#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
-//  showtime("Time to copy host pinned mem <--> dev mem",&copyhostpinnedtime);   
-//#endif
+#if defined(CUDA_HOST_PINNED_MPI) && defined(KOKKOS_ENABLE_CUDA)
+  showtime("Time to copy host pinned mem <--> dev mem",&copyhostpinnedtime);   
+#endif
   showtime("Time to xchg rhs",&xchgrhstime);
   showtime("Total time in solve",&totalsolvetime);
 #endif

diff --git a/packages/adelus/test/vector_random/cxx_main.cpp b/packages/adelus/test/vector_random/cxx_main.cpp
@@ -210,6 +210,17 @@ int main(int argc, char *argv[])
 #ifdef KOKKOS_ENABLE_CUDA
   int gpu_count;
   cudaGetDeviceCount ( &gpu_count );
+  if (nptile > gpu_count) {
+    if( rank == 0 ) {
+      std::cout << "Request more GPUs than the number of GPUs available "
+                << "to MPI processes (requested: " << nptile 
+                << " vs. available: " << gpu_count 
+                << "). Exit without test." << std::endl;
+    }
+    MPI_Finalize() ;
+    return 0;
+  }
+
   Kokkos::InitArguments args;
   args.num_threads = 0;
   args.num_numa    = 0;

@@ -52,6 +52,7 @@
 #define TESTING_AMESOS2_WITH_TPETRA_REMOVE_UVM
 #if defined(TESTING_AMESOS2_WITH_TPETRA_REMOVE_UVM)
 #include "KokkosKernels_SparseUtils.hpp"
+#include "KokkosKernels_Sorting.hpp"
 #endif
 
 namespace Amesos2 {
@@ -608,7 +609,7 @@ namespace Amesos2 {
     // sort
     if( ordering == SORTED_INDICES ) {
       using execution_space = typename KV_GS::execution_space;
-      KokkosKernels::Impl::sort_crs_matrix <execution_space, KV_GS, KV_GO, KV_S>
+      KokkosKernels::sort_crs_matrix <execution_space, KV_GS, KV_GO, KV_S>
         (rowptr, colind, nzval);
     }
     #endif

@@ -1060,7 +1060,6 @@ Superlu<Matrix,Vector>::triangular_solve_factor()
   if (data_.options.ConditionNumber == SLU::YES) {
     using STM = Teuchos::ScalarTraits<magnitude_type>;
     const magnitude_type eps = STM::eps ();
-    int n = data_.perm_r.extent(0);
 
     SCformat *Lstore = (SCformat*)(data_.L.Store);
     int nsuper = 1 + Lstore->nsuper;
@@ -1077,6 +1076,7 @@ Superlu<Matrix,Vector>::triangular_solve_factor()
     condition_flag = (((double)max_cols * nsuper) * eps * multiply_fact >= data_.rcond);
 
 #ifdef HAVE_AMESOS2_VERBOSE_DEBUG
+    int n = data_.perm_r.extent(0);
     std::cout << this->getComm()->getRank()
               << " : anorm = " << data_.anorm << ", rcond = " << data_.rcond << ", n = " << n
               << ", num super cols = " << nsuper << ", max super cols = " << max_cols

@@ -532,14 +532,14 @@ namespace Amesos2 {
     }
 
     static void gsequ_loc(SLUD::SuperMatrix* A, double* r, double* c, 
-			  double* rowcnd, double* colcnd, double* amax, int* info, 
+			  double* rowcnd, double* colcnd, double* amax, SLUD::int_t* info, 
 			  SLUD::gridinfo_t* grid)
     {
       SLUD::Z::pzgsequ(A, r, c, rowcnd, colcnd, amax, info, grid);
     }
 
     static void gsequ(SLUD::SuperMatrix* A, double* r, double* c, 
-		      double* rowcnd, double* colcnd, double* amax, int* info)
+		      double* rowcnd, double* colcnd, double* amax, SLUD::int_t* info)
     {
       SLUD::Z::zgsequ_dist(A, r, c, rowcnd, colcnd, amax, info);
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -177,7 +177,7 @@ elif [[ "$ATDM_CONFIG_COMPILER" == "CUDA"* ]] ; then @@
     fi
     # Ninja
-    module load ninja/1.7.2
+    #module load ninja/1.7.2
     # CMake
     #module swap cmake/3.6.2 cmake/3.12.3
@@ Expand Down @@