Remove usage of subpackages from TriBITS Kokkos build and downstream Trilinos packages (#11545)

RELEASE_NOTES

@@ -1,3 +1,34 @@
+
+###############################################################################
+#                                                                             #
+# Trilinos Release 14.2 Release Notes                                         #
+#                                                                             #
+###############################################################################
+
+Kokkos
+
+  - Subpackages have been removed Kokkos.  That means that CMake cache
+    variables like Trilinos_ENABLE_KokkosCore or
+    Trilinos_ENABLE_KokkosAlgorithms no longer exist.  Now there is just the
+    top-level package Kokkos enabled with Trilinos_ENABLE_Kokkos.  In
+    addition, all downstream Trilinos packages that had dependencies on Kokkos
+    subpackages now just depend on the parent package Kokkos.  That means that
+    any optional dependencies on a Kokkos subpackage is now an optional
+    dependency on the parent package Kokkos.  For example, the Sacado package
+    had an optional dependency on KokkosCore which could be disabled with -D
+    Sacado_ENABLE_KokkosCore=OFF which now needs to be disabled with -D
+    Sacado_ENABLE_Kokkos=OFF.  In addition, the #define macros for optional
+    dependencies on Kokkos subpackages have changed names.  For example, the
+    Sacado_config.h file defined the preprocessor macros
+    HAVE_SACADO_KOKKOSCORE and HAVE_SACADO_KOKKOSCONTAINERS but now just
+    defines HAVE_SACADO_KOKKOS.  Therefore, downstream C++ code will need to
+    adjust any code that depends on these preprocessor defines.  However,
+    other than changes in the names of the the CMake Cache variables
+    `<Something>_ENABLE_Kokkos<Subpkg>` to `<Something>_ENABLE_Kokkos` and the
+    changes in the names of the proprocessor defines ``XXX_KOKKOS<SUBPKG>`` to
+    ``XXX_KOKKOS``, all other behavior should be identical.
+
+
 ###############################################################################
 #                                                                             #
 # Trilinos Release 14.0 Release Notes                                         #

cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.crs.cmake

@@ -89,7 +89,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
   # Select test disables
-  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing" FORCE)
+  set (Kokkos_CoreUnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing" FORCE)
 
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"

cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.deprecation-removal.cmake

@@ -89,7 +89,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
   # Select test disables
-  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing" FORCE)
+  set (Kokkos_CoreUnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing" FORCE)
 
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"

cmake/std/PullRequestLinuxCuda11.4.2uvmOffDisabledPackagesTestingSettings.cmake

@@ -117,9 +117,9 @@ set (PanzerMiniEM_MiniEM-BlockPrec_Augmentation_MPI_4_DISABLE ON CACHE BOOL "Tem
 set (PanzerMiniEM_MiniEM-BlockPrec_RefMaxwell_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
 set (ROL_example_PDE-OPT_poisson-boltzmann_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
 
-# Disable a couple of unit tests in test KokkosCore_UnitTest_Cuda_MPI_1 that
+# Disable a couple of unit tests in test Kokkos_CoreUnitTest_Cuda_MPI_1 that
 # are randomly failing in PR test iterations (#6799)
-set (KokkosCore_UnitTest_Cuda_MPI_1_EXTRA_ARGS
+set (Kokkos_CoreUnitTest_Cuda_MPI_1_EXTRA_ARGS
   "--gtest_filter=-cuda.debug_pin_um_to_host:cuda.debug_serial_execution"
   CACHE STRING "Temporary disable for CUDA PR testing")
 

cmake/std/PullRequestLinuxCuda11.4.2uvmOffTestingSettings.cmake

@@ -117,9 +117,9 @@ set (MueLu_UnitTestsIntrepid2Tpetra_MPI_4_DISABLE ON CACHE BOOL "Temporary disab
 #set (PanzerMiniEM_MiniEM-BlockPrec_RefMaxwell_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
 set (ROL_example_PDE-OPT_poisson-boltzmann_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
 
-# Disable a couple of unit tests in test KokkosCore_UnitTest_Cuda_MPI_1 that
+# Disable a couple of unit tests in test Kokkos_CoreUnitTest_Cuda_MPI_1 that
 # are randomly failing in PR test iterations (#6799)
-set (KokkosCore_UnitTest_Cuda_MPI_1_EXTRA_ARGS
+set (Kokkos_CoreUnitTest_Cuda_MPI_1_EXTRA_ARGS
   "--gtest_filter=-cuda.debug_pin_um_to_host:cuda.debug_serial_execution"
   CACHE STRING "Temporary disable for CUDA PR testing")
 

cmake/std/atdm/ATDMDisables.cmake

@@ -253,16 +253,16 @@ ATDM_SET_ENABLE(ROL_adapters_tpetra_test_vector_SimulatedVectorTpetraBatchManage
 ATDM_SET_ENABLE(ROL_adapters_tpetra_test_vector_SimulatedVectorTpetraBatchManagerInterface_MPI_4_DISABLE ON)
 
 # Disable Kokkos timing based test. See #8545.
-ATDM_SET_ENABLE(KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON)
+ATDM_SET_ENABLE(Kokkos_CoreUnitTest_CudaTimingBased_MPI_1_DISABLE ON)
 
 # Disable serial.Random_XorShift64 due to random failures. See #3282.
-ATDM_SET_CACHE(KokkosAlgorithms_UnitTest_MPI_1_EXTRA_ARGS
+ATDM_SET_CACHE(Kokkos_AlgorithmsUnitTest_MPI_1_EXTRA_ARGS
   "--gtest_filter=-*Random_XorShift64:-*Random_XorShift1024" CACHE STRING)
 
 IF (ATDM_NODE_TYPE STREQUAL "OPENMP")
 
   # Disable ctest DISABLED test (otherwise, this shows up on CDash as "NotRun")
-  ATDM_SET_ENABLE(KokkosContainers_PerformanceTest_OpenMP_DISABLE ON)
+  ATDM_SET_ENABLE(Kokkos_ContainersPerformanceTest_OpenMP_DISABLE ON)
 
 ENDIF()
 
@@ -272,13 +272,13 @@ IF ("${ATDM_CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
   ATDM_SET_ENABLE(PanzerAdaptersSTK_MixedPoissonExample-ConvTest-Hex-Order-3_DISABLE ON)
 
   # Too expensive for full debug builds after Kokkos 2.99 upgrade
-  ATDM_SET_ENABLE(KokkosCore_UnitTest_Serial_MPI_1_DISABLE ON)
+  ATDM_SET_ENABLE(Kokkos_CoreUnitTest_Serial_MPI_1_DISABLE ON)
   ATDM_SET_ENABLE(KokkosKernels_blas_serial_MPI_1_DISABLE ON)
 
   IF (ATDM_USE_OPENMP)
 
     # Too expensive for full debug builds after Kokkos 2.99 upgrade
-    ATDM_SET_ENABLE(KokkosCore_UnitTest_OpenMP_MPI_1_DISABLE ON)
+    ATDM_SET_ENABLE(Kokkos_CoreUnitTest_OpenMP_MPI_1_DISABLE ON)
     ATDM_SET_ENABLE(KokkosKernels_blas_openmp_MPI_1_DISABLE ON)
 
   ENDIF()
@@ -313,11 +313,11 @@ IF (ATDM_NODE_TYPE STREQUAL "CUDA")
   ATDM_SET_ENABLE(Zoltan_ch_simple_zoltan_parallel_DISABLE ON)
 
   # Disable ctest DISABLED test (otherwise, this shows up on CDash as "NotRun")
-  ATDM_SET_ENABLE(KokkosContainers_PerformanceTest_Cuda_DISABLE ON)
+  ATDM_SET_ENABLE(Kokkos_ContainersPerformanceTest_Cuda_DISABLE ON)
 
-  # Disable a couple of unit tests in test KokkosCore_UnitTest_Cuda_MPI_1
+  # Disable a couple of unit tests in test Kokkos_CoreUnitTest_Cuda_MPI_1
   # (#6799)
-  ATDM_SET_CACHE(KokkosCore_UnitTest_Cuda_MPI_1_EXTRA_ARGS
+  ATDM_SET_CACHE(Kokkos_CoreUnitTest_Cuda_MPI_1_EXTRA_ARGS
     "--gtest_filter=-cuda.debug_pin_um_to_host:cuda.debug_serial_execution"
     CACHE STRING )
 
@@ -326,9 +326,9 @@ IF (ATDM_NODE_TYPE STREQUAL "CUDA")
   # that this Kokkos inter operability test runs in serial, we ensure that the
   # GPU ID is not swapped upon initialize resulting in an invalid Cuda stream.
   # See #8544.
-  ATDM_SET_ENABLE(KokkosCore_UnitTest_CudaInterOpStreams_MPI_1_SET_RUN_SERIAL ON)
+  ATDM_SET_ENABLE(Kokkos_CoreUnitTest_CudaInterOpStreams_MPI_1_SET_RUN_SERIAL ON)
   # See #8543.
-  ATDM_SET_ENABLE(KokkosCore_UnitTest_CudaInterOpInit_MPI_1_SET_RUN_SERIAL ON)
+  ATDM_SET_ENABLE(Kokkos_CoreUnitTest_CudaInterOpInit_MPI_1_SET_RUN_SERIAL ON)
 
   # Attempt to address intermittent timeouts reported in #6805
   ATDM_SET_ENABLE(KokkosKernels_sparse_cuda_MPI_1_SET_RUN_SERIAL ON)

cmake/std/atdm/contributed/weaver/tweaks/CUDA-9.2_DEBUG_CUDA_POWER9_VOLTA70.cmake

@@ -2,7 +2,7 @@
 ATDM_SET_ENABLE(TeuchosNumerics_DISABLE_STEQR_TEST ON)
 
 # Disable timing out unit test in this debug build (#3336)
-ATDM_SET_ENABLE(KokkosContainers_UnitTest_Serial_MPI_1_DISABLE ON)
+ATDM_SET_ENABLE(Kokkos_ContainersUnitTest_Serial_MPI_1_DISABLE ON)
 
 # Disable some unit tests that run too slow in this DEBUG build (#2827)
 ATDM_SET_CACHE(KokkosKernels_sparse_serial_MPI_1_EXTRA_ARGS

commonTools/refactoring/refactors/README.md

@@ -0,0 +1,2 @@
+This directory contains refactoring scripts that can't be put in individual
+Trilinos package directories for some reason.

commonTools/refactoring/refactors/remove_kokkos_subpackages_change_test_names_r.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# Script to update the names of Kokkos tests after the refactoring to remove
+# subpackages.
+#
+# Run this script from the base directory and it will update all files.
+#
+# Warning!  Do not run this in a directory where the file
+# remove_kokkos_subpackages_change_test_names.token-list will be found or this
+# will change the names there too and destroy this script!
+#
+# Also, do not create a symbolic link to this script, just use a
+# absolute or relative path from where it is defined in the
+# Trilinos source tree.
+
+# Get the directory for this scirpt which will give us the Trilinos base
+# directory
+_SCRIPT_DIR=`echo $0 | sed "s/\(.*\)\/.*\.sh/\1/g"`
+_TRILINOS_HOME=$_SCRIPT_DIR/../../..
+
+# Run the replacements on all of the files found in subdirectories
+find . -type f \
+  -exec $_TRILINOS_HOME/commonTools/refactoring/string-replace-list.pl \
+  $_SCRIPT_DIR/remove_kokkos_subpackages_change_test_names_r.token-list \
+  '{}' '{}' ';'

commonTools/refactoring/refactors/remove_kokkos_subpackages_change_test_names_r.token-list

@@ -0,0 +1,4 @@
+KokkosCore_UnitTest                 Kokkos_CoreUnitTest
+KokkosContainers_UnitTest           Kokkos_ContainersUnitTest
+KokkosAlgorithms_UnitTest           Kokkos_AlgorithmsUnitTest
+KokkosContainers_PerformanceTest    Kokkos_ContainersPerformanceTest

commonTools/refactoring/refactors/remove_kokkos_subpackages_from_trilinos_packages_r.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Run this script from the Trilinos/packages/ subdir to remove the usage of Kokkos subpackages.
+#
+# Usage:
+#
+#   $ cd Trilinos/packages/
+#   $ ../commonTools/refactoring/refactors/remove_kokkos_subpackages_from_trilinos_packages.sh
+# 
+
+pkgsToRefactor=$(ls | grep -v "kokkos$" | grep -v framework | grep -v TrilinosInstallTests | grep -v new_package)
+
+for pkg in ${pkgsToRefactor} ; do
+
+  if [[ -d ${pkg} ]] ; then
+
+    echo
+    echo "***"
+    echo "*** Refactoring ${pkg}"
+    echo "***"
+    echo
+
+    cd ${pkg}
+
+    time ../../commonTools/refactoring/refactors/remove_kokkos_subpackages_r.sh
+
+    cd ..
+
+  fi
+
+done

commonTools/refactoring/refactors/remove_kokkos_subpackages_r.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Run this in a subdir to strip out usage of Kokkos subpackages for TriBITS
+# packages downstream from the Kokkos package (4/19/2023).
+#
+# Run as:
+#
+#   $ cd <some-base-dir>
+#   $ <this-script-dir>/remove_kokkos_subpackages_r.sh
+#
+# NOTE: Do **NOT** run this script on the packages/kokkos/subdirectory itself
+# (or undo the changes if you do by accident).
+#
+# NOTE: Some manual modifications may be desired after running this script
+# like removing duplicate if clauses like:
+#
+#   if (<Package>_ENABLE_KokkosCore AND <Package>_ENABLE_KokkosContainers ...)
+#
+# becoming:
+#
+#   if (<Package>_ENABLE_Kokkos AND <Package>_ENABLE_Kokkos ...)
+#
+# which should be manually refactored to be
+#
+#   if (<Package>_ENABLE_Kokkos ...)
+#
+# NOTE: This script can't be in the packages/kokkos/ directory because the
+# next Kokos snapshot would delete it and this refactoring is really
+# Trilinos-specific and therefore does not need to be in the external kokkos
+# repo.
+#
+
+_SCRIPT_DIR=`echo $0 | sed "s/\(.*\)\/.*remove_kokkos_subpackages_r.sh/\1/g"`
+#echo $_SCRIPT_DIR
+
+refactDir=${_SCRIPT_DIR}/..
+
+kokkosSubpkgs=(Core Containers Algorithms Simd)
+
+for kokkosSubpkg in ${kokkosSubpkgs[@]}; do
+
+  echo
+  echo "Replace Kokkos${kokkosSubpkg} with Kokkos in Dependencies.cmake file ..."
+  echo
+
+  find . -name Dependencies.cmake -exec ${refactDir}/token-replace.pl Kokkos${kokkosSubpkg} Kokkos {} {} \;
+
+  echo
+  echo "Replace Kokkos${kokkosSubpkg}_ with Kokkos_ in CMakeLists.txt and *.cmake files ..."
+  echo
+
+  find . \( -name CMakeLists.txt -or -name "*.cmake" \) -exec ${refactDir}/string-replace.pl Kokkos${kokkosSubpkg}_ Kokkos_ {} {} \;
+
+  echo
+  echo "Replace _Kokkos${kokkosSubpkg} with _Kokkos in CMakeLists.txt and *.cmake files ..."
+  echo
+
+  find . \( -name CMakeLists.txt -or -name "*.cmake" \) -exec ${refactDir}/string-replace.pl _Kokkos${kokkosSubpkg} _Kokkos {} {} \;
+
+  echo
+  echo "Replace _KOKKOS${kokkosSubpkg^^} with _KOKKOS in all files ..."
+  echo
+
+  find . -type f -exec ${refactDir}/string-replace.pl _KOKKOS${kokkosSubpkg^^} _KOKKOS {} {} \;
+
+done
+
+echo
+echo "Replace duplicates of 'Kokkos' in Dependencies.cmake file ..."
+echo
+
+find . -name Dependencies.cmake -exec sed -i 's/Kokkos Kokkos\([ )\n]\)/Kokkos\1/g' {} \; 
+find . -name Dependencies.cmake -exec sed -i 's/Kokkos Kokkos\([ )\n]\)/Kokkos\1/g' {} \; 
+find . -name Dependencies.cmake -exec sed -i 's/Kokkos Kokkos\([ )\n]\)/Kokkos\1/g' {} \; 

commonTools/refactoring/string-replace-list.pl

@@ -4,22 +4,32 @@
 # Here it is allowed for file_in and
 # file_out to be the same file.
 #
+
 use strict;
-#
+
 my $g_use_msg =
   "Use: string-replace-list.pl string_file file_in file_out\n";
 if( scalar(@ARGV) != 3 ) {
   print STDERR $g_use_msg;
   exit(-1);
 }
-#
+
 my $string_file   = shift;
 my $file_in_name  = shift;
 my $file_out_name = shift;
-#
+
 open STRING_NAMES_FILE, "<$string_file" || die "The file $string_file could not be opened!";
-my @string_names = <STRING_NAMES_FILE>;
-#
+my @string_names_lines = <STRING_NAMES_FILE>;
+my @string_names;
+foreach(@string_names_lines) {
+  my $line = $_;
+  my @string_names_line_array = split(' ', $line);
+  foreach(@string_names_line_array) {
+    chomp($_);
+    push @string_names, $_
+  }
+}
+
 #print "file_in_name = $file_in_name\n";
 if($file_in_name=~/CVS/) {
 #  print "Do not replace in CVS\n";
@@ -28,24 +38,27 @@
 open FILE_IN, "<$file_in_name";
 my @file_in_array = <FILE_IN>;
 close FILE_IN;
-#
+
 #print $match_str . "\n";
-#
+
 my @file_out_array;
 my $did_replacement = 0;
 foreach(@file_in_array) {
   my $line = $_;
+  #print "\n";
+  #print "----------------------------------------------------\n";
+  #print "\n";
   #print $line;
   my $i;
-  for( $i=0; $i <scalar(@string_names); $i += 3 ) {
+  for( $i=0; $i <scalar(@string_names); $i += 2 ) {
     my ($find_string, $replace_string) = ($string_names[$i], $string_names[$i+1]);
     defined($find_string) || die $!;
     defined($replace_string) || die $!;
     chomp($find_string);
-    chomp($replace_string);
-    #print "string_names line = $_";
     #print "find string = ${find_string}\n";
+    chomp($replace_string);
     #print "replace string = ${replace_string}\n";
+    #print "string_names line = $_";
     $did_replacement = 1 if $line=~s/$find_string/$replace_string/g;
     #print $line;
   }

packages/compadre/cmake/Dependencies.cmake

@@ -1,6 +1,6 @@
 tribits_package_define_dependencies(
   LIB_REQUIRED_PACKAGES
-    KokkosCore KokkosContainers KokkosAlgorithms KokkosKernels
+    Kokkos KokkosKernels
   LIB_OPTIONAL_TPLS
     MPI CUDA
   )