Enable all PT packages for ATDM-based PR cuda build, turn off MueLu and Xpetra Experimental, explicitly disable RDC for ATDM builds

ProjectName.cmake

@@ -1,10 +1,8 @@
 SET(PROJECT_NAME Trilinos)
 
-SET(${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE  ON  CACHE  BOOL
-  "Set by default in Trilinos ProjectName.cmake." )
+SET(${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT  ON)
 
-SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE  ON CACHE BOOL
-  "Set by default in Trilinos ProjectName.cmake." )
+SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT  ON)
 
 IF ( WIN32 AND NOT CYGWIN )
   MESSAGE(STATUS "Warning: Setting ${PROJECT_NAME}_ENABLE_Fortran=OFF by default"

cmake/ctest/drivers/atdm/README.md

@@ -23,7 +23,6 @@ The base directory:
 contains files that are used for driving builds on all machines.  These files
 define common behavior and reduces duplication to ease maintenance.
 
-
 This directory contains the file:
 
 ```
@@ -40,27 +39,26 @@ given in the files defined in the directory:
 
 That file reads `JOB_NAME` from the env and uses it to set the CDash build
 name.  (Therefore, the Jenkikns `JOB_NAME` is the same as the CDash build name
-for all of these Trilinos ATDM builds.)
-
+for all of these Trilinos ATDM builds.)  It also other CMake and CTest options
+that are pulled out of the env set by the
+`cmake/std/atdm/<system_name>/environment.sh` script.  (See `$ENV{<varName>}`
+to see what vars are pulled out of the env.)
 
-That directory contains a CTest -S driver script:
+This directory contains a CTest -S driver script:
 
 ```
   Trilinos/cmake/ctest/drivers/atdm/ctest-driver.cmake
 ```
 
-This files sets other CMake and CTest options that are pulled out of the env
-set by the `cmake/std/atdm/<system_name>/environment.sh` script.
-
-Given this file `atdm/ctest-driver.cmake`, then:
+which is called using:
 
 ```
   ctest -S <base-dir>/Trilinos/cmake/ctest/drivers/atdm/ctest-driver.cmake
 ```
 
-can be run using any way desired and it will clone a new Trilinos git repo (if
-not cloned already).  (But this file get directly run by the universal driver
-script `ctest-s-driver.sh` described above.)
+which can be run using any way desired and it will clone a new Trilinos git
+repo (if not cloned already).  (But this file get directly run by the
+universal driver script `ctest-s-driver.sh` described above.)
 
 This directory contains the file:
 

cmake/ctest/drivers/atdm/TrilinosCTestDriverCore.atdm.cmake

@@ -41,6 +41,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   SET( CTEST_NOTES_FILES
     ${CTEST_NOTES_FILES}
     "${TRIBITS_PROJECT_ROOT}/cmake/std/atdm/$ENV{ATDM_CONFIG_KNOWN_SYSTEM_NAME}/environment.sh"
+    "${THIS_FILE_LIST_DIR}/$ENV{ATDM_CONFIG_KNOWN_SYSTEM_NAME}/drivers/$ENV{JOB_NAME}.sh"
     "${CMAKE_CURRENT_LIST_FILE}"
     )
 
@@ -61,15 +62,29 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   SET(CTEST_SITE "$ENV{ATDM_CONFIG_KNOWN_HOSTNAME}")
 
   # See if to enable all of the packages
-
   MESSAGE("ENV ATDM_CONFIG_ENABLE_ALL_PACKAGE = $ENV{ATDM_CONFIG_ENABLE_ALL_PACKAGES}")
-
   IF ($ENV{ATDM_CONFIG_ENABLE_ALL_PACKAGES})
     SET(ATDM_ENABLE_ALL_PACKAGES TRUE)
   ELSE()
     SET(ATDM_ENABLE_ALL_PACKAGES FALSE)
   ENDIF()
 
+  # Get set of options files (must be relative to base Trilinos dir!)
+  MESSAGE("ENV ATDM_CONFIG_CONFIGURE_OPTIONS_FILES = $ENV{ATDM_CONFIG_CONFIGURE_OPTIONS_FILES}")
+  IF (NOT "$ENV{ATDM_CONFIG_CONFIGURE_OPTIONS_FILES}" STREQUAL "")
+    SET(ATDM_CONFIGURE_OPTIONS_FILES $ENV{ATDM_CONFIG_CONFIGURE_OPTIONS_FILES})
+  ELSE()
+    SET(ATDM_CONFIGURE_OPTIONS_FILES cmake/std/atdm/ATDMDevEnv.cmake)
+  ENDIF()
+  PRINT_VAR(ATDM_CONFIGURE_OPTIONS_FILES)
+
+  MESSAGE("Include the configure options files at the top level to influence what package get enabled or disabled ...")
+  FOREACH (CONFIG_OPTIONS_FILE ${ATDM_CONFIGURE_OPTIONS_FILES})
+    SET(CONFIG_OPTIONS_FILE "${TRIBITS_PROJECT_ROOT}/${CONFIG_OPTIONS_FILE}")
+    MESSAGE("Including ${CONFIG_OPTIONS_FILE} ...")
+    INCLUDE("${CONFIG_OPTIONS_FILE}")
+  ENDFOREACH()
+
   # See if "panzer" is in the job name
   STRING(REGEX MATCH "panzer" ATDM_PANZER_IN_JOB_NAME
     "${CTEST_BUILD_NAME}" )
@@ -126,19 +141,10 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
     # other packages are not actively developed and are at less of a risk to be
     # broken.
   ENDIF()
-
-  IF (NOT ATDM_ENABLE_ALL_PACKAGES)
-    # Disable the packages that are never used by the ATDM APP codes
-    INCLUDE("${THIS_FILE_LIST_DIR}/../../../std/atdm/ATDMDisables.cmake")
-    SET(ATDM_DISABLES_FILE_ARG ",cmake/std/atdm/ATDMDisables.cmake")
-  ELSE()
-    MESSAGE("NOT reading in ATDMDisables.cmake!")
-    SET(ATDM_DISABLES_FILE_ARG "")
-  ENDIF()
 
   # Point to the ATDM Trilinos configuration
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
-    "-DTrilinos_CONFIGURE_OPTIONS_FILE:STRING=cmake/std/atdm/ATDMDevEnvSettings.cmake${ATDM_DISABLES_FILE_ARG}"
+    "-DTrilinos_CONFIGURE_OPTIONS_FILE:STRING=${ATDM_CONFIGURE_OPTIONS_FILES}"
     "-DTrilinos_TRACE_ADD_TEST=ON"
     "-DTrilinos_ENABLE_CONFIGURE_TIMING=ON"
     ${PANZER_DISABLE_EXAMPLES_ARGS}

cmake/ctest/drivers/atdm/ride/drivers/Trilinos-atdm-white-ride-cuda-debug-pt-all-at-once.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 #export Trilinos_TRACK=ATDM
 export ATDM_CONFIG_ENABLE_ALL_PACKAGES=TRUE
-export Trilinos_ENABLE_SECONDARY_TESTED_CODE=OFF
+#export Trilinos_ENABLE_SECONDARY_TESTED_CODE=OFF
+export ATDM_CONFIG_CONFIGURE_OPTIONS_FILES=cmake/std/atdm/ATDMDevEnvAllPtPackages.cmake
 export Trilinos_CTEST_DO_ALL_AT_ONCE=TRUE
 export Trilinos_CTEST_USE_NEW_AAO_FEATURES=ON
 $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ride/local-driver.sh

cmake/std/atdm/ATDMDevEnvAllPtPackages.cmake

@@ -0,0 +1,20 @@
+###############################################################################
+#
+#              ATDM Configuration for all Trilinos packages
+#
+###############################################################################
+
+INCLUDE("${CMAKE_CURRENT_LIST_DIR}/utils/ATDMDevEnvUtils.cmake")
+
+# Only enable Primary Tested packages for this build
+ATDM_SET_ENABLE(${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE OFF)
+
+# Need Fortran to build some SEACAS code
+ATDM_SET_ENABLE(${PROJECT_NAME}_ENABLE_Fortran ON)
+
+# Don't have stuff for Matio and some SEACAS subpackage has required
+# dependency on this
+ATDM_SET_ENABLE(TPL_ENABLE_Matio OFF)
+
+# Now include the rest of the settings
+INCLUDE("${CMAKE_CURRENT_LIST_DIR}/ATDMDevEnvSettings.cmake")

cmake/std/atdm/ATDMDevEnvSettings.cmake

@@ -114,6 +114,12 @@ ATDM_SET_CACHE(CMAKE_CXX_FLAGS "$ENV{EXTRA_CXX_FLAGS}" CACHE STRING)
 # E) Set up other misc options
 #
 
+# Currently, EMPIRE configures of Trilinos have this enabled by default.  But
+# really we should elevate every subpackage that ATDM uses to Primary Tested.
+# That is the right solution.
+ATDM_SET_ENABLE(Trilinos_ENABLE_SECONDARY_TESTED_CODE ON)
+
+# Other various options
 ATDM_SET_CACHE(CMAKE_SKIP_RULE_DEPENDENCY ON CACHE BOOL)
 ATDM_SET_CACHE(CMAKE_VERBOSE_MAKEFILE OFF CACHE BOOL)
 ATDM_SET_CACHE(MPI_EXEC ${ATDM_MPI_EXEC} CACHE FILEPATH)
@@ -129,19 +135,18 @@ ATDM_SET_CACHE(Trilinos_ENABLE_OpenMP "${ATDM_USE_OPENMP}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_OpenMP "${ATDM_USE_OPENMP}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Pthread "${ATDM_USE_PTHREADS}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Cuda_UVM "${ATDM_USE_CUDA}" CACHE BOOL)
+ATDM_SET_CACHE(Kokkos_ENABLE_Cuda_Relocatable_Device_Code OFF CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_CXX11_DISPATCH_LAMBDA ON CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Cuda_Lambda "${ATDM_USE_CUDA}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Debug_Bounds_Check "${ATDM_BOUNDS_CHECK}" CACHE BOOL)
 ATDM_SET_CACHE(KOKKOS_ENABLE_DEBUG "${ATDM_BOUNDS_CHECK}" CACHE BOOL)
 ATDM_SET_CACHE(KOKKOS_ARCH "$ENV{ATDM_CONFIG_KOKKOS_ARCH}" CACHE STRING)
 ATDM_SET_CACHE(EpetraExt_ENABLE_HDF5 OFF CACHE BOOL)
-ATDM_SET_CACHE(MueLu_ENABLE_Experimental ON CACHE BOOL)
 ATDM_SET_CACHE(MueLu_ENABLE_Epetra OFF CACHE BOOL)
 ATDM_SET_CACHE(Panzer_ENABLE_FADTYPE "Sacado::Fad::DFad<RealType>" CACHE STRING)
 ATDM_SET_CACHE(Phalanx_KOKKOS_DEVICE_TYPE "${ATDM_NODE_TYPE}" CACHE STRING)
 ATDM_SET_CACHE(Phalanx_SHOW_DEPRECATED_WARNINGS OFF CACHE BOOL)
 ATDM_SET_CACHE(Tpetra_INST_CUDA "${ATDM_USE_CUDA}" CACHE BOOL)
-ATDM_SET_CACHE(Xpetra_ENABLE_Experimental ON CACHE BOOL)
 ATDM_SET_CACHE(DART_TESTING_TIMEOUT 600 CACHE STRING)
 
 #

cmake/std/atdm/utils/ATDMDevEnvUtils.cmake

@@ -7,9 +7,9 @@ MACRO(ATDM_SET_CACHE VAR_NAME DEFAULT_VAL CACHE_STR DATA_TYPE)
 ENDMACRO()
 
 MACRO(ATDM_SET_ENABLE  VAR_NAME  VAR_DEFAULT)
-  IF ("${${VAR_DEFAULT}}" STREQUAL "")
+  IF ("${${VAR_NAME}}" STREQUAL "")
     MESSAGE("-- " "Setting default ${VAR_NAME}=${VAR_DEFAULT}")
-    SET(${VAR_NAME} ${VAR_DEFAULT} CACHE BOOL
+    SET(${VAR_NAME} "${VAR_DEFAULT}" CACHE BOOL
       "Set in ${CMAKE_CURRENT_LIST_FILE}")
   ENDIF()
   IF (ATDM_CONFIG_VERBOSE_DEBUG)

cmake/tribits/ci_support/checkin-test.py

@@ -1419,8 +1419,8 @@ def main(cmndLineArgs):
       cmndLineSplit = cmndLineArg.split("=")
       if cmndLineSplit[0].strip() == "--log-file":
         logFileName = cmndLineSplit[1].strip()
-    logFile = open(logFileName, "w", buffering=1) # 1 == line buffering
-    # NOTE: Above, we need to set biffering=1 to make sure that each line that
+    logFile = open(logFileName, "w", buffering=0) # 0 == no buffering
+    # NOTE: Above, we need to set biffering=0 to make sure that each line that
     # gets written is written out the file.  The is critical so that the user
     # can to a `tail -f <log-file>` to see what is going one.
   else:

cmake/tribits/core/package_arch/TribitsGlobalMacros.cmake

@@ -539,7 +539,13 @@ MACRO(TRIBITS_DEFINE_GLOBAL_OPTIONS_AND_DEFINE_EXTRA_REPOS)
     CACHE BOOL
     "Excluded disabled packages from the CPack-generated distribution.")
 
-  ADVANCED_SET( ${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE OFF CACHE BOOL
+  IF ("${${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT}" STREQUAL "")
+    SET(${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT OFF)
+  ENDIF()
+  ADVANCED_SET(
+    ${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE
+    ${${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT}
+    CACHE BOOL
     "Allow Secondary Tested (ST) packages and code to be implicitly enabled." )
 
   IF ("${${PROJECT_NAME}_TEST_CATEGORIES_DEFAULT}" STREQUAL "")
@@ -552,7 +558,13 @@ MACRO(TRIBITS_DEFINE_GLOBAL_OPTIONS_AND_DEFINE_EXTRA_REPOS)
     )
   TRIBITS_GET_INVALID_CATEGORIES(${PROJECT_NAME}_TEST_CATEGORIES)
 
-  ADVANCED_SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE OFF CACHE BOOL
+  IF ("${${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT}" STREQUAL "" )
+    SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT OFF)
+  ENDIF()
+  ADVANCED_SET(
+    ${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE
+    ${${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT}
+    CACHE BOOL
     "Generate a <ProjectName>RepoVersion.txt file.")
 
   IF ("${DART_TESTING_TIMEOUT_DEFAULT}"  STREQUAL "")

cmake/tribits/ctest_driver/TribitsCTestDriverCore.cmake

@@ -571,8 +571,6 @@ INCLUDE(${CMAKE_CURRENT_LIST_DIR}/TribitsCTestDriverCoreHelpers.cmake)
 #     ``CTEST_ENABLE_MODIFIED_PACKAGES_ONLY=TRUE`` is set in which case the
 #     default value is ``TRUE``.
 #
-#   .. _${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE:
-#
 #   ``${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE``
 #
 #     If set to ``TRUE``, then ST packages will get enabled in automated logic
@@ -1441,7 +1439,11 @@ FUNCTION(TRIBITS_CTEST_DRIVER)
   SET_DEFAULT_AND_FROM_ENV( CTEST_SUBMIT_RETRY_COUNT 5 ) # Default defined by ctest 
   SET_DEFAULT_AND_FROM_ENV( CTEST_SUBMIT_RETRY_DELAY 3 ) # Default defined by ctest?
 
-  SET_DEFAULT_AND_FROM_ENV( ${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE OFF )
+  IF ("${${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT}" STREQUAL "")
+    SET(${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT OFF)
+  ENDIF()
+  SET_DEFAULT_AND_FROM_ENV( ${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE
+    ${${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT} )
 
   # List of additional packages that will be enabled over the current set of
   # all packages (that would be set by ${PROJECT_NAME}_ENABLE_ALL_PACKAGES).

cmake/tribits/doc/developers_guide/TribitsDevelopersGuide.rst

@@ -5933,13 +5933,12 @@ extra repos
   configure will fail.
 
 3) If using git as the VC tool, then set the variable
-`${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE`_ in the
+``${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT`` in the
 `<projectDir>/ProjectName.cmake`_ file
 
   For example::
 
-    SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE  TRUE
-      CACHE  BOOL  "Set in ProjectName.cmake")
+    SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT  TRUE)
 
 4) If wanting a clone tool with git repos, set up a link to the
 `clone_extra_repos.py`_ script in the base ``<projectDir>/`` directory
@@ -7873,8 +7872,10 @@ a given TriBITS project are:
 * `${PROJECT_NAME}_ENABLE_EXPORT_MAKEFILES`_
 * `${PROJECT_NAME}_ENABLE_Fortran`_
 * `${PROJECT_NAME}_ENABLE_INSTALL_CMAKE_CONFIG_FILES`_
+* `${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE`_
 * `${PROJECT_NAME}_EXCLUDE_DISABLED_SUBPACKAGES_FROM_DISTRIBUTION`_
 * `${PROJECT_NAME}_GENERATE_EXPORT_FILE_DEPENDENCIES`_
+* `${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE`_
 * `${PROJECT_NAME}_INSTALL_LIBRARIES_AND_HEADERS`_
 * `${PROJECT_NAME}_MUST_FIND_ALL_TPL_LIBS`_
 * `${PROJECT_NAME}_REQUIRES_PYTHON`_
@@ -8198,6 +8199,20 @@ These options are described below.
   ``<PackageName>Config.cmake`` files if it was only installing and providing
   executables (see `${PROJECT_NAME}_INSTALL_LIBRARIES_AND_HEADERS`_).
 
+.. _${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE:
+
+**${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE**
+
+  If ``${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE`` is ``ON``, then packages
+  and subpackages marked as ``ST`` in the `<repoDir>/PackagesList.cmake`_ file
+  will be implicitly enabled along with the ``PT`` packages.  Additional code
+  and tests may also be enabled using this option.  The TriBITS default is
+  ``OFF`` but this can be changed by setting::
+
+    SET(${PROJECT_NAME}_ENABLE_SECONDARY_TESTED_CODE_DEFAULT ON)
+
+  in the `<projectDir>/ProjectName.cmake`_ file.
+
 .. _${PROJECT_NAME}_EXCLUDE_DISABLED_SUBPACKAGES_FROM_DISTRIBUTION:
 
 **${PROJECT_NAME}_EXCLUDE_DISABLED_SUBPACKAGES_FROM_DISTRIBUTION**
@@ -8211,6 +8226,20 @@ These options are described below.
 
     SET(${PROJECT_NAME}_EXCLUDE_DISABLED_SUBPACKAGES_FROM_DISTRIBUTION_DEFAULT FALSE)
 
+.. _${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE:
+
+**${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE**
+
+  If ``${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE`` is ``ON``, then the file
+  ``<Project>RepoVersion.txt`` will get generated as a byproduct of
+  configuring with CMake.  See `Multi-Repository Support`_ and
+  `<Project>_GENERATE_REPO_VERSION_FILE`_.  The default is ``OFF`` but the
+  project can change that by setting::
+
+    SET(${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE_DEFAULT ON)
+
+  in the `<projectDir>/ProjectName.cmake`_ file.
+
 .. _${PROJECT_NAME}_GENERATE_EXPORT_FILE_DEPENDENCIES:
 
 **${PROJECT_NAME}_GENERATE_EXPORT_FILE_DEPENDENCIES**
@@ -8853,7 +8882,7 @@ Below is a snapshot of the output from ``install_devtools.py --help``.
 
 .. _<Project>_EXTRA_REPOSITORIES: TribitsBuildReference.html#project-extra-repositories
 
-.. _${PROJECT_NAME}_GENERATE_REPO_VERSION_FILE: TribitsBuildReference.html#generating-a-project-repo-version-file
+.. _<Project>_GENERATE_REPO_VERSION_FILE: TribitsBuildReference.html#generating-a-project-repo-version-file
 
 .. _Creating a tarball of the source tree: TribitsBuildReference.html#creating-a-tarball-of-the-source-tree