From 55816b5c1009bb5635e457b21b37958e85886210 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Tue, 12 Nov 2024 10:46:47 -0800 Subject: [PATCH 1/5] fixes for builds with no raja --- src/serac/infrastructure/CMakeLists.txt | 2 +- src/serac/infrastructure/debug_print.hpp | 8 +++++ .../functional/domain_integral_kernels.hpp | 2 ++ .../functional/element_restriction.cpp | 36 +++++++++++++++---- .../functional/element_restriction.hpp | 4 +++ 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/serac/infrastructure/CMakeLists.txt b/src/serac/infrastructure/CMakeLists.txt index dc88eed574..53e2fa324a 100644 --- a/src/serac/infrastructure/CMakeLists.txt +++ b/src/serac/infrastructure/CMakeLists.txt @@ -41,7 +41,7 @@ set(infrastructure_sources terminator.cpp ) -set(infrastructure_depends axom::inlet axom::fmt axom::cli11 mfem ${serac_device_depends}) +set(infrastructure_depends axom::inlet axom::fmt axom::cli11 camp mfem ${serac_device_depends}) blt_list_append(TO infrastructure_depends ELEMENTS tribol IF TRIBOL_FOUND) blt_list_append(TO infrastructure_depends ELEMENTS caliper adiak::adiak IF SERAC_ENABLE_PROFILING) list(APPEND infrastructure_depends blt::mpi) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 485f058afa..09c28f3580 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -74,7 +74,11 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) * @param filename the name of the output file */ template +#ifdef SERAC_USE_RAJA void write_to_file(axom::Array arr, std::string filename) +#else +void write_to_file(axom::Array arr, std::string filename) +#endif { std::ofstream outfile(filename); @@ -97,7 +101,11 @@ void write_to_file(axom::Array arr, std::string f * @param filename the name of the output file */ template +#ifdef SERAC_USE_RAJA void write_to_file(axom::Array arr, std::string filename) +#else +void write_to_file(axom::Array arr, std::string filename) +#endif { std::ofstream outfile(filename); diff --git a/src/serac/numerics/functional/domain_integral_kernels.hpp b/src/serac/numerics/functional/domain_integral_kernels.hpp index c218025fbe..b6043464c9 100644 --- a/src/serac/numerics/functional/domain_integral_kernels.hpp +++ b/src/serac/numerics/functional/domain_integral_kernels.hpp @@ -10,7 +10,9 @@ #include "serac/numerics/functional/quadrature_data.hpp" #include "serac/numerics/functional/function_signature.hpp" #include "serac/numerics/functional/differentiate_wrt.hpp" +#ifdef SERAC_USE_RAJA #include "RAJA/RAJA.hpp" +#endif #include #include diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index 9068da4056..7d3a4fa243 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -213,8 +213,12 @@ std::vector > geom_local_face_dofs(int p) return output; } -axom::Array GetElementRestriction(const mfem::FiniteElementSpace* fes, - mfem::Geometry::Type geom) +#ifdef SERAC_USE_RAJA +axom::Array +#else +axom::Array +#endif +GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type geom) { std::vector elem_dofs{}; mfem::Mesh* mesh = fes->GetMesh(); @@ -267,17 +271,29 @@ axom::Array GetElementRestriction(const mfem::F } if (n == 0) { +#ifdef SERAC_USE_RAJA return axom::Array{}; +#else + return axom::Array{}; +#endif } else { - uint64_t dofs_per_elem = elem_dofs.size() / n; + uint64_t dofs_per_elem = elem_dofs.size() / n; +#ifdef SERAC_USE_RAJA axom::Array output(n, dofs_per_elem); +#else + axom::Array output(n, dofs_per_elem); +#endif std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -axom::Array GetFaceDofs(const mfem::FiniteElementSpace* fes, - mfem::Geometry::Type face_geom, FaceType type) +#ifdef SERAC_USE_RAJA +axom::Array +#else +axom::Array +#endif +GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, FaceType type) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -378,10 +394,18 @@ axom::Array GetFaceDofs(const mfem::FiniteEleme delete face_to_elem; if (n == 0) { +#ifdef SERAC_USE_RAJA return axom::Array{}; +#else + return axom::Array{}; +#endif } else { - uint64_t dofs_per_face = face_dofs.size() / n; + uint64_t dofs_per_face = face_dofs.size() / n; +#ifdef SERAC_USE_RAJA axom::Array output(n, dofs_per_face); +#else + axom::Array output(n, dofs_per_face); +#endif std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index 7f12dca598..10aa3ecb97 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -195,7 +195,11 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space +#ifdef SERAC_USE_RAJA axom::Array dof_info; +#else + axom::Array dof_info; +#endif /// whether the underlying dofs are arranged "byNodes" or "byVDim" mfem::Ordering::Type ordering; From 12debca4f424cc70625cbb141afd468423e645a8 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Tue, 12 Nov 2024 11:02:35 -0800 Subject: [PATCH 2/5] change raja to umpire where appropriate --- src/serac/infrastructure/debug_print.hpp | 4 ++-- .../numerics/functional/element_restriction.cpp | 12 ++++++------ .../numerics/functional/element_restriction.hpp | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 09c28f3580..99daf67129 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -74,7 +74,7 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) * @param filename the name of the output file */ template -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE void write_to_file(axom::Array arr, std::string filename) #else void write_to_file(axom::Array arr, std::string filename) @@ -101,7 +101,7 @@ void write_to_file(axom::Array arr, std::string filename) * @param filename the name of the output file */ template -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE void write_to_file(axom::Array arr, std::string filename) #else void write_to_file(axom::Array arr, std::string filename) diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index 7d3a4fa243..03ee48a97e 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -213,7 +213,7 @@ std::vector > geom_local_face_dofs(int p) return output; } -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array #else axom::Array @@ -271,14 +271,14 @@ GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type } if (n == 0) { -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE return axom::Array{}; #else return axom::Array{}; #endif } else { uint64_t dofs_per_elem = elem_dofs.size() / n; -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array output(n, dofs_per_elem); #else axom::Array output(n, dofs_per_elem); @@ -288,7 +288,7 @@ GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type } } -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array #else axom::Array @@ -394,14 +394,14 @@ GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, delete face_to_elem; if (n == 0) { -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE return axom::Array{}; #else return axom::Array{}; #endif } else { uint64_t dofs_per_face = face_dofs.size() / n; -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array output(n, dofs_per_face); #else axom::Array output(n, dofs_per_face); diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index 10aa3ecb97..daeb5220b3 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -195,7 +195,7 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array dof_info; #else axom::Array dof_info; From a07d20cfb76638a8bd805c184d13d0e64d5c7491 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Thu, 14 Nov 2024 09:10:47 -0800 Subject: [PATCH 3/5] use dynamic memory for all --- src/serac/infrastructure/debug_print.hpp | 8 ----- .../functional/element_restriction.cpp | 34 +++---------------- .../functional/element_restriction.hpp | 4 --- 3 files changed, 4 insertions(+), 42 deletions(-) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 99daf67129..9d77c0ee65 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -74,11 +74,7 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) * @param filename the name of the output file */ template -#ifdef SERAC_USE_UMPIRE -void write_to_file(axom::Array arr, std::string filename) -#else void write_to_file(axom::Array arr, std::string filename) -#endif { std::ofstream outfile(filename); @@ -101,11 +97,7 @@ void write_to_file(axom::Array arr, std::string filename) * @param filename the name of the output file */ template -#ifdef SERAC_USE_UMPIRE -void write_to_file(axom::Array arr, std::string filename) -#else void write_to_file(axom::Array arr, std::string filename) -#endif { std::ofstream outfile(filename); diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index 03ee48a97e..58f776bb28 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -213,12 +213,7 @@ std::vector > geom_local_face_dofs(int p) return output; } -#ifdef SERAC_USE_UMPIRE -axom::Array -#else -axom::Array -#endif -GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type geom) +axom::Array GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type geom) { std::vector elem_dofs{}; mfem::Mesh* mesh = fes->GetMesh(); @@ -271,29 +266,16 @@ GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type } if (n == 0) { -#ifdef SERAC_USE_UMPIRE - return axom::Array{}; -#else return axom::Array{}; -#endif } else { - uint64_t dofs_per_elem = elem_dofs.size() / n; -#ifdef SERAC_USE_UMPIRE - axom::Array output(n, dofs_per_elem); -#else + uint64_t dofs_per_elem = elem_dofs.size() / n; axom::Array output(n, dofs_per_elem); -#endif std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -#ifdef SERAC_USE_UMPIRE -axom::Array -#else -axom::Array -#endif -GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, FaceType type) +axom::Array GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, FaceType type) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -394,18 +376,10 @@ GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, delete face_to_elem; if (n == 0) { -#ifdef SERAC_USE_UMPIRE - return axom::Array{}; -#else return axom::Array{}; -#endif } else { - uint64_t dofs_per_face = face_dofs.size() / n; -#ifdef SERAC_USE_UMPIRE - axom::Array output(n, dofs_per_face); -#else + uint64_t dofs_per_face = face_dofs.size() / n; axom::Array output(n, dofs_per_face); -#endif std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index daeb5220b3..3dc1e6a3a3 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -195,11 +195,7 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space -#ifdef SERAC_USE_UMPIRE - axom::Array dof_info; -#else axom::Array dof_info; -#endif /// whether the underlying dofs are arranged "byNodes" or "byVDim" mfem::Ordering::Type ordering; From b132020b8a539de15f5acb1e36748c90bb32aca6 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Thu, 14 Nov 2024 16:09:42 -0800 Subject: [PATCH 4/5] copy arr if needed --- src/serac/infrastructure/debug_print.hpp | 65 +++++++++++------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 9d77c0ee65..70ed1a3ee9 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -68,53 +68,48 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) } /** - * @brief write a 2D array of values out to file, in a space-separated format + * @brief write an array of values out to file, in a space-separated format * @tparam T the type of each value in the array * @param v the values to write to file * @param filename the name of the output file */ -template -void write_to_file(axom::Array arr, std::string filename) +template +void write_to_file(const ArrayT& arr_orig, std::string filename) { std::ofstream outfile(filename); - for (axom::IndexType i = 0; i < arr.shape()[0]; i++) { - outfile << "{"; - for (axom::IndexType j = 0; j < arr.shape()[1]; j++) { - outfile << arr(i, j); - if (j < arr.shape()[1] - 1) outfile << ", "; - } - outfile << "}\n"; - } - - outfile.close(); -} - -/** - * @brief write a 3D array of values out to file, in a mathematica-compatible format - * @tparam T the type of each value in the array - * @param v the values to write to file - * @param filename the name of the output file - */ -template -void write_to_file(axom::Array arr, std::string filename) -{ - std::ofstream outfile(filename); +#ifdef SERAC_USE_UMPIRE + axom::Array arr(arr_orig); +#else + const auto& arr = arr_orig; +#endif - outfile << std::setprecision(16); + if constexpr(ArrayT::Dims == 2) + { + for (axom::IndexType i = 0; i < arr.shape()[0]; i++) { + outfile << "{"; + for (axom::IndexType j = 0; j < arr.shape()[1]; j++) { + outfile << arr(i, j); + if (j < arr.shape()[1] - 1) outfile << ", "; + } + outfile << "}\n"; + } + } else if constexpr(ArrayT::Dims == 3) { + outfile << std::setprecision(16); - for (axom::IndexType i = 0; i < arr.shape()[0]; i++) { - outfile << "{"; - for (axom::IndexType j = 0; j < arr.shape()[1]; j++) { + for (axom::IndexType i = 0; i < arr.shape()[0]; i++) { outfile << "{"; - for (axom::IndexType k = 0; k < arr.shape()[2]; k++) { - outfile << arr(i, j, k); - if (k < arr.shape()[2] - 1) outfile << ", "; + for (axom::IndexType j = 0; j < arr.shape()[1]; j++) { + outfile << "{"; + for (axom::IndexType k = 0; k < arr.shape()[2]; k++) { + outfile << arr(i, j, k); + if (k < arr.shape()[2] - 1) outfile << ", "; + } + outfile << "}"; + if (j < arr.shape()[1] - 1) outfile << ", "; } - outfile << "}"; - if (j < arr.shape()[1] - 1) outfile << ", "; + outfile << "}\n"; } - outfile << "}\n"; } outfile.close(); From 3028760910bb54c9361c4f6d2aa61aa4a870fa71 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Thu, 14 Nov 2024 16:21:52 -0800 Subject: [PATCH 5/5] separate if statements --- src/serac/infrastructure/debug_print.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 70ed1a3ee9..5dcc79b4ca 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -84,8 +84,7 @@ void write_to_file(const ArrayT& arr_orig, std::string filename) const auto& arr = arr_orig; #endif - if constexpr(ArrayT::Dims == 2) - { + if constexpr (ArrayT::Dims == 2) { for (axom::IndexType i = 0; i < arr.shape()[0]; i++) { outfile << "{"; for (axom::IndexType j = 0; j < arr.shape()[1]; j++) { @@ -94,7 +93,8 @@ void write_to_file(const ArrayT& arr_orig, std::string filename) } outfile << "}\n"; } - } else if constexpr(ArrayT::Dims == 3) { + } + if constexpr (ArrayT::Dims == 3) { outfile << std::setprecision(16); for (axom::IndexType i = 0; i < arr.shape()[0]; i++) {