Update mfem petsc/ slepc smoketests to use GPU on CUDA builds

src/tests/mfem_petsc_smoketest.cpp

@@ -407,21 +407,19 @@ int ex1_main(int argc, char *argv[])
 TEST(MfemPetscSmoketest, MfemPetscEx1)
 {
   ::testing::internal::CaptureStdout();
-  // https://github.com/LLNL/serac/issues/1158
-  // #ifdef SERAC_USE_CUDA
-  //   const char* fake_argv[] = {"ex1",
-  //                              "-m",
-  //                              SERAC_REPO_DIR "/mfem/data/star.mesh",
-  //                              "--usepetsc",
-  //                              "--partial-assembly",
-  //                              "--device",
-  //                              "cuda",
-  //                              "--petscopts",
-  //                              SERAC_REPO_DIR "/mfem/examples/petsc/rc_ex1p_device"};
-  // #else
+#ifdef SERAC_USE_CUDA
+  const char* fake_argv[] = {"ex1",
+                             "-m",
+                             SERAC_REPO_DIR "/mfem/data/star.mesh",
+                             "--usepetsc",
+                             "--device",
+                             "cuda",
+                             "--petscopts",
+                             SERAC_REPO_DIR "/src/tests/rc_mfem_petsc_smoketest_gpu"};
+#else
   const char* fake_argv[] = {"ex1",        "-m",          SERAC_REPO_DIR "/mfem/data/amr-quad.mesh",
-                             "--usepetsc", "--petscopts", SERAC_REPO_DIR "/mfem/examples/petsc/rc_ex1p"};
-  // #endif
+                             "--usepetsc", "--petscopts", SERAC_REPO_DIR "/src/tests/rc_mfem_petsc_smoketest"};
+#endif
   int fake_argc = sizeof(fake_argv) / sizeof(fake_argv[0]);
   ex1_main(fake_argc, const_cast<char**>(fake_argv));
   std::string output = ::testing::internal::GetCapturedStdout();

src/tests/mfem_slepc_smoketest.cpp

@@ -461,21 +461,19 @@ int ex11_main(int argc, char *argv[])
 TEST(MfemSlepcSmoketest, MfemPetscEx11)
 {
   ::testing::internal::CaptureStdout();
-  // https://github.com/LLNL/serac/issues/1158
-  // #ifdef SERAC_USE_CUDA
-  //   const char* fake_argv[] = {"ex11",       "-m",          SERAC_REPO_DIR "/mfem/data/star.mesh",
-  //                              "--useslepc", "--slepcopts", SERAC_REPO_DIR
-  //                              "/mfem/examples/petsc/rc_ex11p_lobpcg_device",
-  //                              "--device",   "cuda",        "--no-visualization"};
-  // #else
+#ifdef SERAC_USE_CUDA
+  const char* fake_argv[] = {"ex11",       "-m",          SERAC_REPO_DIR "/mfem/data/star.mesh",
+                             "--useslepc", "--slepcopts", SERAC_REPO_DIR "/src/tests/rc_mfem_slepc_smoketest_gpu",
+                             "--device",   "cuda",        "--no-visualization"};
+#else
   const char* fake_argv[] = {"ex11",
                              "-m",
                              SERAC_REPO_DIR "/mfem/data/star.mesh",
                              "--useslepc",
                              "--slepcopts",
-                             SERAC_REPO_DIR "/mfem/examples/petsc/rc_ex11p_lobpcg",
+                             SERAC_REPO_DIR "/src/tests/rc_mfem_slepc_smoketest",
                              "--no-visualization"};
-  // #endif
+#endif
   int fake_argc = sizeof(fake_argv) / sizeof(fake_argv[0]);
   ex11_main(fake_argc, const_cast<char**>(fake_argv));
   std::string output = ::testing::internal::GetCapturedStdout();

src/tests/rc_mfem_petsc_smoketest

@@ -0,0 +1,19 @@
+# Source: mfem/examples/petsc/rc_ex1p
+# Options for the Krylov solver
+-ksp_view
+-ksp_converged_reason
+# Options for the preconditioner
+-pc_type gamg
+-pc_gamg_type agg
+-pc_gamg_agg_nsmooths 1
+-pc_gamg_coarse_eq_limit 100
+-pc_gamg_reuse_interpolation
+-pc_gamg_square_graph 1
+-pc_gamg_threshold 0.0
+-mg_levels_ksp_max_it 2
+-mg_levels_ksp_type chebyshev
+-mg_levels_esteig_ksp_type cg
+-mg_levels_esteig_ksp_max_it 10
+-mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
+-pc_gamg_use_sa_esteig 0
+-mg_levels_pc_type sor

src/tests/rc_mfem_petsc_smoketest_gpu

@@ -0,0 +1,28 @@
+# Source: mfem/examples/petsc/rc_ex1p_deviceamg
+-log_view
+# Options for the Krylov solver
+-ksp_view
+-ksp_converged_reason
+# Options for the preconditioner
+-pc_type gamg
+-pc_gamg_type agg
+-pc_gamg_agg_nsmooths 1
+-pc_gamg_coarse_eq_limit 100
+-pc_gamg_reuse_interpolation
+-pc_gamg_square_graph 1
+-pc_gamg_threshold 0.0
+-mg_levels_ksp_max_it 2
+-mg_levels_ksp_type chebyshev
+# SERAC_EDIT_START
+# -mg_levels_esteig_ksp_type cg
+# -mg_levels_esteig_ksp_max_it 10
+# SERAC_EDIT_END
+-mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
+#sor is not implemented for GPU, use jacobi
+-mg_levels_pc_type jacobi
+
+
+# SERAC_EDIT_START
+# Disable "gpu aware mpi" to avoid PETSc runtime errors
+-use_gpu_aware_mpi 0
+# SERAC_EDIT_END

src/tests/rc_mfem_slepc_smoketest

@@ -0,0 +1,12 @@
+# Source: mfem/examples/petsc/rc_ex11p_lobpcg
+# Options for the eigenvalue solver
+-eps_monitor
+-eps_converged_reason
+-eps_view_values
+-eps_type lobpcg
+-eps_gen_hermitian
+-eps_smallest_real
+-eps_lobpcg_blocksize 5
+# Options for the spectral transform
+-st_type precond
+-st_pc_type gamg

src/tests/rc_mfem_slepc_smoketest_gpu

@@ -0,0 +1,34 @@
+# Source: mfem/examples/petsc/rc_ex11p_lobpcg_device
+-log_view
+# Options for the eigenvalue solver
+-eps_view
+-eps_monitor
+-eps_converged_reason
+-eps_view_values
+-eps_type lobpcg
+-eps_gen_hermitian
+-eps_smallest_real
+-eps_lobpcg_blocksize 5
+# Options for the spectral transform
+-st_type precond
+-prefix_push st_
+-pc_type gamg
+-pc_gamg_type agg
+-pc_gamg_agg_nsmooths 1
+-pc_gamg_coarse_eq_limit 100
+-pc_gamg_reuse_interpolation
+-pc_gamg_square_graph 1
+-pc_gamg_threshold 0.0
+-mg_levels_ksp_max_it 2
+-mg_levels_ksp_type chebyshev
+-mg_levels_esteig_ksp_type cg
+-mg_levels_esteig_ksp_max_it 10
+-mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
+#sor is not implemented for GPU, use jacobi
+-mg_levels_pc_type jacobi
+-prefix_pop
+
+# SERAC_EDIT_START
+# Disable "gpu aware mpi" to avoid PETSc runtime errors
+-use_gpu_aware_mpi 0
+# SERAC_EDIT_END