diff --git a/cmake/blt b/cmake/blt index b7314a86e..66378271f 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit b7314a86e9fc78baf2682ad55509bbc8acd4bce6 +Subproject commit 66378271f605de0b628ab93196fd41f5c7b59452 diff --git a/scripts/llnl/build_devtools.py b/scripts/llnl/build_devtools.py index 532c486c6..9c9ceb1cd 100755 --- a/scripts/llnl/build_devtools.py +++ b/scripts/llnl/build_devtools.py @@ -18,8 +18,6 @@ from argparse import ArgumentParser -import os - def parse_args(): "Parses args from command line" diff --git a/scripts/llnl/common_build_functions.py b/scripts/llnl/common_build_functions.py index f1e9ddaa1..927aacf17 100755 --- a/scripts/llnl/common_build_functions.py +++ b/scripts/llnl/common_build_functions.py @@ -676,6 +676,10 @@ def get_shared_libs_dir(): return pjoin(get_shared_base_dir(), "libs", get_project_name()) +def get_shared_spot_dir(): + return pjoin(get_shared_base_dir(), "califiles") + + def get_uberenv_path(): return pjoin(get_script_dir(), "../uberenv/uberenv.py") diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index 81f5a51fa..7f5ea83a1 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -30,8 +30,8 @@ Introduction to SPOT LLNL for vizualizing performance data. SPOT is an external tool and does not need to be linked into Serac. -Build Instructions ------------------- +TPL Build Instructions +---------------------- To use Adiak and Caliper with Serac, install the ``profiling`` variant of ``serac`` with Spack, i.e., ``serac+profiling``. Note that these libraries are pre-built as @@ -109,3 +109,21 @@ of this file, use `cali-query `_), and open the directory containing one or more ``.cali`` files. For more information, watch this recorded `tutorial `_. +Benchmarking Serac +------------------ + +To run all of Serac's benchmarks in one command, first make sure Serac is configured +with benchmarking enabled (off by default). Then, run the build target ``run_benchmarks``. + +.. code-block:: bash + + ./config-build.py -hc -DENABLE_BENCHMARKS=ON + cd + make -j + make run_benchmarks + find . -name "*.cali" -print0 | xargs -0 mv -t . + pwd + +This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a Caliper file for +each benchmark run. The ``find`` command afterwards ensures all Caliper files are moved to the same directory. Now, you +can visualize the results with SPOT, entering the path printed from ``pwd``. diff --git a/src/serac/infrastructure/tests/profiling.cpp b/src/serac/infrastructure/tests/profiling.cpp index 8571f48b9..e38e0ef9d 100644 --- a/src/serac/infrastructure/tests/profiling.cpp +++ b/src/serac/infrastructure/tests/profiling.cpp @@ -47,6 +47,8 @@ TEST(Profiling, MeshRefinement) pmesh->UniformRefinement(); } + // Add metadata + SERAC_SET_METADATA("test", "profiling"); SERAC_SET_METADATA("mesh_file", mesh_file.c_str()); SERAC_SET_METADATA("number_mesh_elements", pmesh->GetNE()); diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt index b5c866a0b..7eb946021 100644 --- a/src/serac/physics/benchmarks/CMakeLists.txt +++ b/src/serac/physics/benchmarks/CMakeLists.txt @@ -4,16 +4,28 @@ # # SPDX-License-Identifier: (BSD-3-Clause) -set(benchmark_depends serac_physics) +set(physics_benchmark_depends serac_physics) -blt_add_executable(NAME benchmark_thermal - SOURCES benchmark_thermal.cpp - DEPENDS_ON ${benchmark_depends} - OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks - FOLDER serac/benchmarks) +set(physics_benchmark_targets + physics_benchmark_functional + physics_benchmark_solid_nonlinear_solve + physics_benchmark_thermal + ) -blt_add_executable(NAME benchmark_functional - SOURCES benchmark_functional.cpp - DEPENDS_ON ${benchmark_depends} - OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks - FOLDER serac/benchmarks) +# Create executable for each benchmark +foreach(physics_benchmark ${physics_benchmark_targets}) + blt_add_executable(NAME ${physics_benchmark} + SOURCES ${physics_benchmark}.cpp + DEPENDS_ON ${physics_benchmark_depends} + OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks + FOLDER serac/benchmarks + ) + + # Add benchmarks with various task counts + foreach(task_count 1 4 16) + blt_add_benchmark(NAME ${physics_benchmark}_${task_count}_task_count + COMMAND ${physics_benchmark} + NUM_MPI_TASKS ${task_count} + ) + endforeach() +endforeach() diff --git a/src/serac/physics/benchmarks/benchmark_functional.cpp b/src/serac/physics/benchmarks/physics_benchmark_functional.cpp similarity index 98% rename from src/serac/physics/benchmarks/benchmark_functional.cpp rename to src/serac/physics/benchmarks/physics_benchmark_functional.cpp index 0127e64ea..6645074b0 100644 --- a/src/serac/physics/benchmarks/benchmark_functional.cpp +++ b/src/serac/physics/benchmarks/physics_benchmark_functional.cpp @@ -86,6 +86,9 @@ int main(int argc, char* argv[]) // Initialize profiling serac::profiling::initialize(); + // Add metadata + SERAC_SET_METADATA("test", "functional"); + SERAC_MARK_BEGIN("scalar H1"); SERAC_MARK_BEGIN("dimension 2, order 1"); diff --git a/src/serac/physics/tests/solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp similarity index 71% rename from src/serac/physics/tests/solid_nonlinear_solve.cpp rename to src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp index 4a46f9a69..23e98abb6 100644 --- a/src/serac/physics/tests/solid_nonlinear_solve.cpp +++ b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp @@ -7,14 +7,14 @@ #include "serac/physics/solid_mechanics.hpp" #include "serac/physics/solid_mechanics_contact.hpp" -#include +#include #include +#include +#include #include #include -#include #include "axom/slic/core/SimpleLogger.hpp" -#include #include "mfem.hpp" #include "serac/physics/materials/liquid_crystal_elastomer.hpp" @@ -28,30 +28,62 @@ using namespace serac; -std::string mesh_path = "."; - -enum Prec +enum class Prec { JACOBI, STRUMPACK, CHOLESKI, LU, MULTIGRID, - PETSC_MULTIGRID + PETSC_MULTIGRID, + NONE }; -enum NonlinSolve +enum class NonlinSolve { NEWTON, LINESEARCH, CRITICALPOINT, - TRUSTREGION + TRUSTREGION, + NONE }; -NonlinSolve nonlinSolve = NonlinSolve::TRUSTREGION; -Prec prec = Prec::JACOBI; +std::string mesh_path = "."; +// string->value matching for optionally entering options as string in command line +std::map precMap = { + {"jacobi", Prec::JACOBI}, {"strumpack", Prec::STRUMPACK}, {"choleski", Prec::CHOLESKI}, + {"lu", Prec::LU}, {"multigrid", Prec::MULTIGRID}, {"petsc_multigrid", Prec::PETSC_MULTIGRID}, + {"none", Prec::NONE}, +}; +std::map nonlinSolveMap = { + {"newton", NonlinSolve::NEWTON}, + {"linesearch", NonlinSolve::LINESEARCH}, + {"critialpoint", NonlinSolve::CRITICALPOINT}, + {"trustregion", NonlinSolve::TRUSTREGION}, + {"none", NonlinSolve::NONE}, +}; + +const std::string precToString(Prec prec) +{ + for (const auto& p : precMap) { + if (p.second == prec) { + return p.first; + } + } + return "unknown"; +} -auto get_opts(int max_iters, double abs_tol = 1e-9) +const std::string nonlinSolveToString(NonlinSolve nonlinSolve) +{ + for (const auto& n : nonlinSolveMap) { + if (n.second == nonlinSolve) { + return n.first; + } + } + return "unknown"; +} + +auto get_opts(NonlinSolve nonlinSolve, Prec prec, int max_iters, double abs_tol = 1e-9) { serac::NonlinearSolverOptions nonlinear_options{ .nonlin_solver = NonlinearSolver::TrustRegion, @@ -108,6 +140,10 @@ auto get_opts(int max_iters, double abs_tol = 1e-9) nonlinear_options.nonlin_solver = NonlinearSolver::TrustRegion; break; } + case NonlinSolve::NONE: + default: { + SLIC_ERROR_ROOT("invalid nonlinear solver specified"); + } } switch (prec) { @@ -149,18 +185,16 @@ auto get_opts(int max_iters, double abs_tol = 1e-9) linear_options.petsc_preconditioner = PetscPCType::HMG; break; } + case Prec::NONE: default: { - SLIC_ERROR_ROOT("error, invalid preconditioner specified"); + SLIC_ERROR_ROOT("invalid preconditioner specified"); } } return std::make_pair(nonlinear_options, linear_options); } -#include -#include - -void functional_solid_test_euler() +void functional_solid_test_euler(NonlinSolve nonlinSolve, Prec prec) { // initialize serac axom::sidre::DataStore datastore; @@ -192,7 +226,7 @@ void functional_solid_test_euler() // solid mechanics using seracSolidType = serac::SolidMechanics>; - auto [nonlinear_options, linear_options] = get_opts(3 * Nx * Ny * Nz, 1e-9); + auto [nonlinear_options, linear_options] = get_opts(nonlinSolve, prec, 3 * Nx * Ny * Nz, 1e-9); auto seracSolid = std::make_unique( nonlinear_options, linear_options, serac::solid_mechanics::default_quasistatic_options, @@ -224,7 +258,7 @@ void functional_solid_test_euler() } } -void functional_solid_test_nonlinear_buckle(double loadMagnitude) +void functional_solid_test_nonlinear_buckle(NonlinSolve nonlinSolve, Prec prec, double loadMagnitude) { // initialize serac axom::sidre::DataStore datastore; @@ -258,7 +292,7 @@ void functional_solid_test_nonlinear_buckle(double loadMagnitude) // solid mechanics using seracSolidType = serac::SolidMechanics>; - auto [nonlinear_options, linear_options] = get_opts(3 * Nx * Ny * Nz, 1e-11); + auto [nonlinear_options, linear_options] = get_opts(nonlinSolve, prec, 3 * Nx * Ny * Nz, 1e-11); auto seracSolid = std::make_unique( nonlinear_options, linear_options, serac::solid_mechanics::default_quasistatic_options, @@ -280,30 +314,64 @@ void functional_solid_test_nonlinear_buckle(double loadMagnitude) seracSolid->outputStateToDisk("paraview_buckle_easy"); } -TEST(SolidMechanics, nonlinear_solve_buckle_easy) { functional_solid_test_nonlinear_buckle(5e-10); } -// TEST(SolidMechanics, nonlinear_solve_buckle_medium) { functional_solid_test_nonlinear_buckle(4e-4); } -// TEST(SolidMechanics, nonlinear_solve_buckle_hard) { functional_solid_test_nonlinear_buckle(3e-2); } -// TEST(SolidMechanics, nonlinear_solve_euler) { functional_solid_test_euler(); } - int main(int argc, char* argv[]) { - axom::CLI::App app{"Nonlinear problems"}; - // app.add_option("-p", mesh_path, "Path to mesh files")->check(axom::CLI::ExistingDirectory); - app.add_option("--nonlinear-solver", nonlinSolve, "Nonlinear solver", true); - app.add_option("--preconditioner", prec, "Preconditioner", true); - app.set_help_flag("--help"); - app.allow_extras()->parse(argc, argv); + serac::initialize(argc, argv); - ::testing::InitGoogleTest(&argc, argv); + SERAC_MARK_FUNCTION; - serac::initialize(argc, argv); + NonlinSolve nonlinSolve = NonlinSolve::NONE; + Prec prec = Prec::NONE; + + axom::CLI::App app{"Solid Nonlinear Solve Benchmark"}; + // app.add_option("-m,--mesh", mesh_path, "Path to mesh files")->check(axom::CLI::ExistingDirectory); + app.add_option("-n,--nonlinear-solver", nonlinSolve, "Nonlinear solver") + ->transform(axom::CLI::CheckedTransformer(nonlinSolveMap, axom::CLI::ignore_case)); + app.add_option("-p,--preconditioner", prec, "Preconditioner") + ->transform(axom::CLI::CheckedTransformer(precMap, axom::CLI::ignore_case)); + + // Parse the arguments and check if they are good + try { + CLI11_PARSE(app, argc, argv); + } catch (const axom::CLI::ParseError& e) { + serac::logger::flush(); + if (e.get_name() == "CallForHelp") { + auto msg = app.help(); + SLIC_INFO_ROOT(msg); + serac::exitGracefully(); + } else { + auto err_msg = axom::CLI::FailureMessage::simple(&app, e); + SLIC_ERROR_ROOT(err_msg); + } + } SERAC_SET_METADATA("test", "solid_nonlinear_solve"); - SERAC_SET_METADATA("nonlinear solver", std::to_string(nonlinSolve)); - SERAC_SET_METADATA("preconditioner", std::to_string(prec)); - int result = RUN_ALL_TESTS(); - serac::exitGracefully(result); + // If you do not specify preconditioner and nonlinear solver, run the following pre-selected options + if (nonlinSolve == NonlinSolve::NONE && prec == Prec::NONE) { + SERAC_MARK_BEGIN("Jacobi Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::JACOBI, 5e-10); + SERAC_MARK_END("Jacobi Preconditioner"); + + SERAC_MARK_BEGIN("Multigrid Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::MULTIGRID, 5e-10); + SERAC_MARK_END("Multigrid Preconditioner"); + + SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner"); + functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, 5e-10); + SERAC_MARK_END("Petsc Multigrid Preconditioner"); + } else { + SERAC_SET_METADATA("nonlinear solver", nonlinSolveToString(nonlinSolve)); + SERAC_SET_METADATA("preconditioner", precToString(prec)); + + SERAC_MARK_BEGIN("Custom Preconditioner"); + functional_solid_test_nonlinear_buckle(nonlinSolve, prec, 5e-10); + SERAC_MARK_END("Custom Preconditioner"); + } + + // functional_solid_test_nonlinear_buckle(4e-4); + // functional_solid_test_nonlinear_buckle(3e-2); + // functional_solid_test_euler(); - return result; + serac::exitGracefully(0); } diff --git a/src/serac/physics/benchmarks/benchmark_thermal.cpp b/src/serac/physics/benchmarks/physics_benchmark_thermal.cpp similarity index 100% rename from src/serac/physics/benchmarks/benchmark_thermal.cpp rename to src/serac/physics/benchmarks/physics_benchmark_thermal.cpp diff --git a/src/serac/physics/tests/CMakeLists.txt b/src/serac/physics/tests/CMakeLists.txt index 69bbdf5c3..698378cd1 100644 --- a/src/serac/physics/tests/CMakeLists.txt +++ b/src/serac/physics/tests/CMakeLists.txt @@ -39,7 +39,6 @@ set(physics_parallel_test_sources dynamic_thermal_adjoint.cpp solid_reaction_adjoint.cpp thermal_nonlinear_solve.cpp - solid_nonlinear_solve.cpp ) blt_list_append(TO physics_parallel_test_sources ELEMENTS contact_patch.cpp contact_beam.cpp IF TRIBOL_FOUND)