diff --git a/Version.cmake b/Version.cmake index 9faac3380623..b1deb052e815 100644 --- a/Version.cmake +++ b/Version.cmake @@ -59,10 +59,10 @@ # for release mode and set the version. # -SET(Trilinos_VERSION 13.1) +SET(Trilinos_VERSION 13.5) SET(Trilinos_MAJOR_VERSION 13) -SET(Trilinos_MAJOR_MINOR_VERSION 130100) -SET(Trilinos_VERSION_STRING "13.1 (Dev)") +SET(Trilinos_MAJOR_MINOR_VERSION 130500) +SET(Trilinos_VERSION_STRING "13.5 (Dev)") SET(Trilinos_ENABLE_DEVELOPMENT_MODE_DEFAULT ON) # Change to 'OFF' for a release # Used by testing scripts and should not be used elsewhere diff --git a/cmake/std/PullRequestLinuxIntel19.0.5TestingSettings.cmake b/cmake/std/PullRequestLinuxIntel19.0.5TestingSettings.cmake index 0249973430b0..d161c3b2b243 100644 --- a/cmake/std/PullRequestLinuxIntel19.0.5TestingSettings.cmake +++ b/cmake/std/PullRequestLinuxIntel19.0.5TestingSettings.cmake @@ -57,18 +57,6 @@ set (ROL_test_step_TrustRegion_MPI_1_DISABLE ON CACHE BOOL "Temporarily disabled set (ROL_tutorial_BoundAndInequality_MPI_1_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -# Tests turned off until an issue with Scotch can be resolved -set (Zoltan2_scotch_example_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_Partitioning1_VWeights_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_Partitioning1_OneProc_VWeights_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_Partitioning1_OneProc_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_Partitioning1_OneProc_EWeights_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_Partitioning1_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_Partitioning1_EWeights_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan2_pamgenMeshAdapterTest_scotch_MPI_4_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Phalanx_dynamic_data_layout_MPI_1_DISABLE ON CACHE BOOL "Temporarily disabled in PR testing") -set (Zoltan_ENABLE_Scotch OFF CACHE BOOL "Temporarily disabled in PR testing") - # Misc options typically added by CI testing mode in TriBITS # Use the below option only when submitting to the dashboard diff --git a/cmake/std/pr_config/pullrequest.ini b/cmake/std/pr_config/pullrequest.ini index 923add11eeb7..35ba1da912de 100644 --- a/cmake/std/pr_config/pullrequest.ini +++ b/cmake/std/pr_config/pullrequest.ini @@ -300,14 +300,10 @@ setenv LDFLAGS: -lifcore [Trilinos_pullrequest_intel_19.0.5] -use SEMS-ARCHIVE-ENV: -use ATDM-ENV: -module-load sems-archive-gcc: 6.1.0 -module-load sems-archive-intel: 19.0.5 -module-load sems-archive-mpich: 3.2 -use SEMS-ARCHIVE-DEFAULT: -module-remove sems-archive-openmpi: -use ATDM-DEFAULT: +module-purge: +module-load sems-intel: 19.0.5 +module-load sems-mpich: 3.2.1 +use SEMS-DEFAULT: setenv LDFLAGS: -lifcore diff --git a/cmake/std/trilinosprhelpers/unittests/trilinos_pr_test.ini b/cmake/std/trilinosprhelpers/unittests/trilinos_pr_test.ini index be14b856f78e..1f981efdff30 100644 --- a/cmake/std/trilinosprhelpers/unittests/trilinos_pr_test.ini +++ b/cmake/std/trilinosprhelpers/unittests/trilinos_pr_test.ini @@ -240,13 +240,11 @@ use ATDM-DEFAULT: [Trilinos_pullrequest_intel_19.0.5] -use SEMS-ARCHIVE-ENV: -use ATDM-ENV: -module-load sems-archive-gcc: 6.1.0 -module-load sems-archive-intel: _19.0.5 -module-load sems-archive-mpich: 3.2 -use SEMS-ARCHIVE-DEFAULT: -use ATDM-DEFAULT: +module-purge: +module-load sems-intel: 19.0.5 +module-load sems-mpich: 3.2.1 +use SEMS-DEFAULT: +setenv LDFLAGS: -lifcore diff --git a/packages/intrepid2/src/Cell/Intrepid2_CellGeometry.hpp b/packages/intrepid2/src/Cell/Intrepid2_CellGeometry.hpp index ed2dbdad09d8..feb58da17d52 100644 --- a/packages/intrepid2/src/Cell/Intrepid2_CellGeometry.hpp +++ b/packages/intrepid2/src/Cell/Intrepid2_CellGeometry.hpp @@ -169,10 +169,10 @@ namespace Intrepid2 SubdivisionStrategy subdivisionStrategy = NO_SUBDIVISION, HypercubeNodeOrdering nodeOrdering = HYPERCUBE_NODE_ORDER_TENSOR); - /** \brief Node-based constructor for straight-edged geometry. + /** \brief Node-based constructor for straight-edged geometry. \param [in] cellTopo - the cell topology for all cells. \param [in] cellToNodes - (C,LN) container specifying the global index of the local node. - \param [in] nodes - (GN,D) container specifying the coordinate weight for the global node in the specified dimension. + \param [in] nodes - (GN,D) container specifying the coordinate weight for the global node in the specified dimension; if cellToNodes is not allocated, this must be a (C,N,D) container \param [in] claimAffine - whether to assume (without checking) that the mapping from reference space is affine. (If claimAffine is false, we check whether the cell topology is simplicial, and if so, set the affine_ member variable to true.) \param [in] nodeOrdering - applicable for hypercube cell topologies; specifies whether to use the order used by Shards (and lowest-order Intrepid2 bases), or the one used by higher-order Intrepid2 bases. */ diff --git a/packages/intrepid2/src/Cell/Intrepid2_CellGeometryDef.hpp b/packages/intrepid2/src/Cell/Intrepid2_CellGeometryDef.hpp index 8b97008325f4..452b345d9af1 100644 --- a/packages/intrepid2/src/Cell/Intrepid2_CellGeometryDef.hpp +++ b/packages/intrepid2/src/Cell/Intrepid2_CellGeometryDef.hpp @@ -513,9 +513,19 @@ namespace Intrepid2 cellToNodes_(cellToNodes), nodes_(nodes) { - numCells_ = cellToNodes.extent_int(0); - numNodesPerCell_ = cellToNodes.extent_int(1); - INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(numNodesPerCell_ != cellTopo.getNodeCount(), std::invalid_argument, "cellToNodes.extent(1) does not match the cell topology node count"); + if(cellToNodes.is_allocated()) + { + numCells_ = cellToNodes.extent_int(0); + numNodesPerCell_ = cellToNodes.extent_int(1); + INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(numNodesPerCell_ != cellTopo.getNodeCount(), std::invalid_argument, "cellToNodes.extent(1) does not match the cell topology node count"); + } + else + { + numCells_ = nodes.extent_int(0); + numNodesPerCell_ = nodes.extent_int(1); + INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(numNodesPerCell_ != cellTopo.getNodeCount(), std::invalid_argument, "nodes.extent(1) does not match the cell topology node count"); + } + if (!claimAffine) { diff --git a/packages/intrepid2/unit-test/MonolithicExecutable/CellGeometryTests.cpp b/packages/intrepid2/unit-test/MonolithicExecutable/CellGeometryTests.cpp index bf87bd8696d5..e38214d18c31 100644 --- a/packages/intrepid2/unit-test/MonolithicExecutable/CellGeometryTests.cpp +++ b/packages/intrepid2/unit-test/MonolithicExecutable/CellGeometryTests.cpp @@ -372,6 +372,35 @@ namespace } } + TEUCHOS_UNIT_TEST( CellGeometry, NodeBased_Constructor ) + { + const int spaceDim = 3; + + using PointScalar = double; + using DeviceType = DefaultTestDeviceType; + using ExecutionSpace = DeviceType::execution_space; + + const auto cellTopo = shards::CellTopology(shards::getCellTopologyData >()); + + using Geometry = CellGeometry; + Geometry::HypercubeNodeOrdering nodeOrdering = Geometry::HYPERCUBE_NODE_ORDER_TENSOR; + + ScalarView cellToNodes; + ScalarView nodes("nodes", 1, 8, spaceDim); + Kokkos::parallel_for("fill nodes for CellGeometry", Kokkos::RangePolicy(0,nodes.extent(1)), + KOKKOS_LAMBDA (const int i) { + nodes(0,i,0) = (i%2 == 1) ? 1. : 0.; + nodes(0,i,1) = ((i/2)%2 == 1) ? 1. : 0.; + nodes(0,i,2) = (i/4 == 1) ? 1. : 0.; + }); + + const bool claimAffine = false; + CellGeometry cellGeometry(cellTopo, cellToNodes, nodes, claimAffine, nodeOrdering); + + TEST_EQUALITY(1, cellGeometry.numCells()); + TEST_EQUALITY(8, cellGeometry.numNodesPerCell()); + } + TEUCHOS_UNIT_TEST( CellGeometry, Jacobian_Uniform_Quads ) { const int spaceDim = 2; diff --git a/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp b/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp index 5527318a5c2d..a92dd3c54b9c 100644 --- a/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp +++ b/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp @@ -1,8 +1,12 @@ #include "Kokkos_Core.hpp" #include "Teuchos_UnitTestHarness.hpp" #include "Phalanx_KokkosViewOfViews.hpp" +#include +// ******************************** // This test demonstrates how to create a view of views for double and FAD types. +// Original implementation +// ******************************** using exec_t = Kokkos::DefaultExecutionSpace; using mem_t = Kokkos::DefaultExecutionSpace::memory_space; @@ -58,6 +62,9 @@ TEUCHOS_UNIT_TEST(PhalanxViewOfViews,OldImpl) { } } +// ******************************** +// New implementation (automatically adds the unmanaged memory trait to inner view) +// ******************************** TEUCHOS_UNIT_TEST(PhalanxViewOfViews,NewImpl) { const int num_cells = 10; @@ -115,3 +122,109 @@ TEUCHOS_UNIT_TEST(PhalanxViewOfViews,NewImpl) { } } + +// ******************************** +// Demonstrates an alternative path for ViewOfViews that uses a user +// defined wrapper and the assignment operator on device to disable +// the reference tracking. +// ******************************** + +// Force this test to always run without UVM. +// For Cuda builds, ignore the default memory space in the Cuda +// execution space since it can be set to UVM in Trilinos +// configure. For non-Cuda builds, just use the default memory space +// in the execution space. +namespace Kokkos { + class Cuda; + class CudaSpace; +} +using DeviceExecutionSpace = Kokkos::DefaultExecutionSpace; +using DeviceMemorySpace = std::conditional::value, + Kokkos::CudaSpace, + Kokkos::DefaultExecutionSpace::memory_space>::type; +using view = Kokkos::View; +using view_host = view::HostMirror; + +class Wrapper { +public: + view a_; + view b_; + KOKKOS_INLINE_FUNCTION + Wrapper() : a_(nullptr,0),b_(nullptr,0) {} + + Wrapper(const view_host& a, const view_host& b) + : a_(a.label(),a.layout()),b_(b.label(),b.layout()) + { + Kokkos::deep_copy(a_,a); + Kokkos::deep_copy(b_,b); + } + + KOKKOS_DEFAULTED_FUNCTION + Wrapper& operator=(const Wrapper& src) = default; + + KOKKOS_INLINE_FUNCTION + double multiply(int i) const {return a_(i) * b_(i);} +}; + +TEUCHOS_UNIT_TEST(PhalanxViewOfViews,WrapperExample) { + + constexpr int num_objects = 3; + constexpr int view_size = 20; + + // This object must exist for lifetime if v_of_uo to keep inner + // views in scope. + std::vector uo(num_objects); + { + view_host a("a",view_size); + view_host b("b",view_size); + view_host c("c",view_size); + view_host d("d",view_size); + view_host e("e",view_size); + view_host f("f",view_size); + + Kokkos::deep_copy(a,0.0); + Kokkos::deep_copy(b,1.0); + Kokkos::deep_copy(c,2.0); + Kokkos::deep_copy(d,3.0); + Kokkos::deep_copy(e,4.0); + Kokkos::deep_copy(f,5.0); + + uo[0] = Wrapper(a,b); + uo[1] = Wrapper(c,d); + uo[2] = Wrapper(e,f); + } + + Kokkos::View v_of_uo("v_of_uo",num_objects); + + for (int i=0; i < 3; ++i) { + Wrapper tmp = uo[i]; + Kokkos::parallel_for("initialize view_of_uo",1,KOKKOS_LAMBDA(const int ) { + // reference counting is disabled on device + v_of_uo(i) = tmp; + }); + Kokkos::fence(); + } + + Kokkos::View results("results",num_objects,view_size); + Kokkos::MDRangePolicy> policy({0,0},{num_objects,view_size}); + Kokkos::parallel_for("v_of_uo",policy,KOKKOS_LAMBDA(const int i,const int j) { + results(i,j) = v_of_uo(i).multiply(j) + static_cast(j); + }); + Kokkos::fence(); + + auto results_host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),results); + + constexpr auto tol = std::numeric_limits::epsilon() * 100.0; + for (int i=0; i < num_objects; ++i) { + for (int j=0; j < view_size; ++j) { + + double gold_value = static_cast(j); + if (i == 1) + gold_value += 6.0; + else if (i == 2) + gold_value += 20.0; + + TEST_FLOATING_EQUALITY(results_host(i,j),gold_value,tol); + } + } +} diff --git a/packages/sacado/test/GTestSuite/googletest/googlemock/docs/cook_book.md b/packages/sacado/test/GTestSuite/googletest/googlemock/docs/cook_book.md index 3fc1198ac830..87bd72ed906a 100644 --- a/packages/sacado/test/GTestSuite/googletest/googlemock/docs/cook_book.md +++ b/packages/sacado/test/GTestSuite/googletest/googlemock/docs/cook_book.md @@ -1425,7 +1425,7 @@ Use `Pair` when comparing maps or other associative containers. using testing::ElementsAre; using testing::Pair; ... - std::map m = {{"a", 1}, {"b", 2}, {"c", 3}}; + std::map m = { {"a", 1}, {"b", 2}, {"c", 3}}; EXPECT_THAT(m, ElementsAre(Pair("a", 1), Pair("b", 2), Pair("c", 3))); ``` diff --git a/packages/trilinoscouplings/examples/scaling/TrilinosCouplings_Statistics.hpp b/packages/trilinoscouplings/examples/scaling/TrilinosCouplings_Statistics.hpp index ba570d4f7c56..1fea0b302a10 100644 --- a/packages/trilinoscouplings/examples/scaling/TrilinosCouplings_Statistics.hpp +++ b/packages/trilinoscouplings/examples/scaling/TrilinosCouplings_Statistics.hpp @@ -144,7 +144,7 @@ class MachineLearningStatistics_Hex3D { int node3 = edgeToNode(edge, 0); int node4 = edgeToNode(edge, 1); - int numElems = elemToEdge.dimension(0); + int numElems = elemToNode.dimension(0); for(int i=0; i +#include +#include #include "TrilinosCouplings_config.h" @@ -167,6 +169,10 @@ #include "stk_mesh/base/FieldBase.hpp" // for field_data, etc #include "stk_mesh/base/Types.hpp" // for BucketVector, EntityId +#ifdef HAVE_TRILINOSCOUPLINGS_MUELU +#include "TrilinosCouplings_Statistics.hpp" +#endif + /*********************************************************/ /* Typedefs */ /*********************************************************/ @@ -481,16 +487,20 @@ int main(int argc, char *argv[]) { for (int i=0; i CoordFieldType; // get coordinates field @@ -537,6 +547,19 @@ int main(int argc, char *argv[]) { } // end loop over mesh parts int numNodesPerElem = cellType.getNodeCount(); +#if defined(HAVE_TRILINOSCOUPLINGS_MUELU) && defined(HAVE_MUELU_EPETRA) + int numEdgesPerElem = cellType.getEdgeCount(); +#endif + + if(MyPID==0) { + std::cout<<"Cell Topology: "< MLStatistics(numGlobalElements); + bool do_statistics = !strcmp(cellType.getName(),"Hexahedron_8"); + std::cout<<"do_statistics = "< elemToNode(numLocalElems,numNodesPerElem); + Intrepid::FieldContainer elemToEdge(numLocalElems,numEdgesPerElem); + Intrepid::FieldContainer nodeCoord (numLocalNodes, spaceDim); + Intrepid::FieldContainer sigmaVal(numLocalElems); + + int elem_ct=0; + std::map,int> local_edge_hash; + std::vector > edge_vector; + + for (size_t bucketIndex = 0; bucketIndex < localElementBuckets.size(); ++bucketIndex) { + stk::mesh::Bucket &elemBucket = *localElementBuckets[bucketIndex]; + for (size_t elemIndex = 0; elemIndex < elemBucket.size(); ++elemIndex) { + stk::mesh::Entity elem = elemBucket[elemIndex]; + //TODO (Optimization) It's assumed all elements are the same type, so this is constant. + //TODO Therefore there's no need to do this everytime. + unsigned numNodes = bulkData.num_nodes(elem); + stk::mesh::Entity const* nodes = bulkData.begin_nodes(elem); + for (unsigned inode = 0; inode < numNodes; ++inode) { + double *coord = stk::mesh::field_data(*coords, nodes[inode]); + int lid = globalMapG.LID((int)bulkData.identifier(nodes[inode]) -1); + elemToNode(elem_ct,inode) = lid; + if(lid != -1) { + nodeCoord(lid,0) = coord[0]; + nodeCoord(lid,1) = coord[1]; + if(spaceDim==3) + nodeCoord(lid,2) = coord[2]; + } + }//end node loop + + auto data = cellType.getCellTopologyData(); + for(unsigned iedge=0; iedgeedge[iedge].node[0]; + int n1 = data->edge[iedge].node[1]; + int lid0 = globalMapG.LID((int)bulkData.identifier(nodes[n0]) -1); + int lid1 = globalMapG.LID((int)bulkData.identifier(nodes[n1]) -1); + if(lid0 != -1 && lid1 != -1) { + int lo = std::min(lid0,lid1); + int hi = std::max(lid0,lid1); + std::pair key(lo,hi); + if (local_edge_hash.find(key) == local_edge_hash.end()) { + int new_edge_id = edge_vector.size(); + local_edge_hash[key] = new_edge_id; + edge_vector.push_back(key); + elemToEdge(elem_ct,iedge) = new_edge_id; + } + else { + elemToEdge(elem_ct,iedge) = local_edge_hash[key]; + } + } + }//end edge loop + + sigmaVal(elem_ct) = 1;// Not doing sigma here + + elem_ct++; + }//end element loop + }//end bucket loop + + Intrepid::FieldContainer edgeToNode(edge_vector.size(), 2); + for(int i=0; i<(int)edge_vector.size(); i++) { + edgeToNode(i,0) = edge_vector[i].first; + edgeToNode(i,1) = edge_vector[i].second; + } + + + MLStatistics.Phase1(elemToNode,elemToEdge,edgeToNode,nodeCoord,sigmaVal); + + } +#endif + /**********************************************************************************/ /******************** DEFINE WORKSETS AND LOOP OVER THEM **************************/ /**********************************************************************************/ @@ -873,6 +970,17 @@ int main(int argc, char *argv[]) { Time.ResetStartTime(); } + + + + /**********************************************************************************/ + /***************************** STATISTICS (Part II) ******************************/ + /**********************************************************************************/ +#if defined(HAVE_TRILINOSCOUPLINGS_MUELU) && defined(HAVE_MUELU_EPETRA) + if(do_statistics) + MLStatistics.Phase2a(worksetJacobDet,worksetCubWeights); +#endif + /**********************************************************************************/ /* Assemble into Global Matrix */ /**********************************************************************************/ @@ -880,6 +988,7 @@ int main(int argc, char *argv[]) { //"WORKSET CELL" loop: local cell ordinal is relative to numLocalElems //JJH runs from 0 to (#local cells - 1) int worksetCellOrdinal = 0; + for (size_t bucketIndex = 0; bucketIndex < localElementBuckets.size(); ++bucketIndex) { stk::mesh::Bucket &elemBucket = *localElementBuckets[bucketIndex]; @@ -910,13 +1019,15 @@ int main(int argc, char *argv[]) { }// end cell row loop + worksetCellOrdinal++; }// end workset cell loop - worksetCellOrdinal++; + } //for (size_t bucketIndex = 0; ... }// end workset loop + StiffMatrix.GlobalAssemble(); StiffMatrix.FillComplete(); rhsVector.GlobalAssemble(); @@ -926,6 +1037,26 @@ int main(int argc, char *argv[]) { << Time.ElapsedTime() << " seconds" << std::endl; Time.ResetStartTime(); } +/**********************************************************************************/ +/***************************** STATISTICS (Part IIb) ******************************/ +/**********************************************************************************/ +#if defined(HAVE_TRILINOSCOUPLINGS_MUELU) && defined(HAVE_MUELU_EPETRA) + if(do_statistics){ + MLStatistics.Phase2b(Teuchos::rcp(&StiffMatrix.Graph(),false),Teuchos::rcp(&nCoord,false)); + } +#endif +/**********************************************************************************/ +/***************************** STATISTICS (Part III) ******************************/ +/**********************************************************************************/ +#if defined(HAVE_TRILINOSCOUPLINGS_MUELU) && defined(HAVE_MUELU_EPETRA) + if(do_statistics){ + MLStatistics.Phase3(); + Teuchos::ParameterList problemStatistics = MLStatistics.GetStatistics(); + if(MyPID==0) std::cout<<"*** Problem Statistics ***"< const Scalar exactSolution(const Scalar& x, const Scalar& y, const Scalar& z) { + // Patch test: tri-linear function is in the FE space and should be recovered + return 1. + x + y + z + x*y + x*z + y*z + x*y*z; + + // Patch test - tet: function is in the FE space and should be recovered - return 1. + x + y + z ; + // return 1. + x + y + z ; // Patch test - hex: tri-linear function is in the FE space and should be recovered // return 1. + x + y + z + x*y + x*z + y*z + x*y*z;