Merge branch 'brhillman/eamxx/scream-upstream-merge-20230405' (#5582)

Bring in changes from downstream SCREAM repo to fix SCREAMv1 build errors.

[BFB]

.gitmodules

@@ -74,3 +74,9 @@
 	path = components/ww3/src/WW3
 	url = [email protected]:E3SM-Project/WW3.git
 	branch = e3sm
+[submodule "externals/haero"]
+	path = externals/haero
+	url = [email protected]:eagles-project/haero.git
+[submodule "externals/mam4xx"]
+	path = externals/mam4xx
+	url = [email protected]:eagles-project/mam4xx.git

cime_config/config_archive.xml

@@ -26,12 +26,10 @@
   </comp_archive_spec>
 
   <comp_archive_spec compname="scream" compclass="atm">
-    <rest_file_extension>[ri]</rest_file_extension>
-    <rest_file_extension>rh\d*</rest_file_extension>
-    <rest_file_extension>rs</rest_file_extension>
-    <hist_file_extension>h\d*.*\.nc$</hist_file_extension>
-    <hist_file_extension>e</hist_file_extension>
-    <rest_history_varname>nhfil</rest_history_varname>
+    <rest_file_extension>r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*</rest_file_extension>
+    <rest_file_extension>rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*</rest_file_extension>
+    <!-- The following matches "hi.AVGTYPE.FREQUNITS_xFREQ.TIMESTAMP.nc"-->
+    <hist_file_extension>hi\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*\.\d{4}-\d{2}-\d{2}-\d{5}\.nc$</hist_file_extension>
   </comp_archive_spec>
 
   <comp_archive_spec compname="elm" compclass="lnd">

cime_config/machines/cmake_macros/gnu_docker.cmake

@@ -0,0 +1,68 @@
+set(AR "/opt/conda/bin/x86_64-conda-linux-gnu-ar")
+set(CFLAGS "-mcmodel=medium")
+if (compile_threaded)
+  string(APPEND CFLAGS " -fopenmp")
+endif()
+if (DEBUG)
+  string(APPEND CFLAGS " -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow")
+endif()
+if (NOT DEBUG)
+  string(APPEND CFLAGS " -O")
+endif()
+if (COMP_NAME STREQUAL csm_share)
+  string(APPEND CFLAGS " -std=c99")
+endif()
+set(CXXFLAGS "-std=c++14")
+if (compile_threaded)
+  string(APPEND CXXFLAGS " -fopenmp")
+endif()
+if (DEBUG)
+  string(APPEND CXXFLAGS " -g -Wall -fbacktrace")
+endif()
+if (NOT DEBUG)
+  string(APPEND CXXFLAGS " -O")
+endif()
+if (COMP_NAME STREQUAL cism)
+  string(APPEND CMAKE_OPTS " -D CISM_GNU=ON")
+endif()
+string(APPEND CMAKE_OPTS " -D CMAKE_AR=/opt/conda/bin/x86_64-conda-linux-gnu-ar")
+string(APPEND CMAKE_OPTS " -DCMAKE_Fortran_COMPILER_RANLIB=/opt/conda/bin/x86_64-conda-linux-gnu-ranlib")
+string(APPEND CMAKE_OPTS " -DCMAKE_C_COMPILER_RANLIB=/opt/conda/bin/x86_64-conda-linux-gnu-ranlib")
+string(APPEND CMAKE_OPTS " -DCMAKE_CXX_COMPILER_RANLIB=/opt/conda/bin/x86_64-conda-linux-gnu-ranlib")
+string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU")
+if (DEBUG)
+  string(APPEND CPPDEFS " -DYAKL_DEBUG")
+endif()
+set(SLIBS "-L/opt/conda/lib -lnetcdf -lnetcdff")
+set(CXX_LIBS "-lstdc++")
+set(CXX_LINKER "FORTRAN")
+set(FC_AUTO_R8 "-fdefault-real-8")
+set(FFLAGS "-I/opt/conda/include -mcmodel=medium -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none")
+if (compile_threaded)
+  string(APPEND FFLAGS " -fopenmp")
+endif()
+if (DEBUG)
+  string(APPEND FFLAGS " -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow")
+endif()
+if (NOT DEBUG)
+  string(APPEND FFLAGS " -O")
+endif()
+set(FFLAGS_NOOPT "-O0")
+set(FIXEDFLAGS "-ffixed-form")
+set(FREEFLAGS "-ffree-form")
+set(HAS_F2008_CONTIGUOUS "FALSE")
+if (compile_threaded)
+  string(APPEND LDFLAGS " -fopenmp")
+endif()
+set(SLIBS " -L/opt/conda/lib -lnetcdff -lnetcdf")
+set(MPI_PATH "/opt/conda")
+set(MPICC "/opt/conda/bin/mpicc")
+set(MPICXX "/opt/conda/bin/mpicxx")
+set(MPIFC "/opt/conda/bin/mpif90")
+set(NETCDF_C_PATH "/opt/conda")
+set(NETCDF_FORTRAN_PATH "/opt/conda")
+set(PNETCDF_PATH "/opt/conda")
+set(SCC "/opt/conda/bin/x86_64-conda-linux-gnu-gcc")
+set(SCXX "/opt/conda/bin/x86_64-conda-linux-gnu-g++")
+set(SFC "/opt/conda/bin/x86_64-conda-linux-gnu-gfortran")
+set(SUPPORTS_CXX "TRUE")

cime_config/tests.py

@@ -519,7 +519,7 @@
             "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1",
             "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero",
             "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1",
-            "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1",
+            "ERS_D_Ln21.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2",
             )
     },
 
@@ -529,7 +529,6 @@
             #  "SMS_D_Ln2.ne30_ne30.F2000-SCREAMv1-AQP1", # Uncomment once IC file for ne30 is ready
             "ERS_Ln22.ne30_ne30.F2010-SCREAMv1",
             "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1",
-            "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1-X6T",
             "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels",
             "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero",
             )

components/cmake/build_eamxx.cmake

@@ -31,8 +31,13 @@ function(build_eamxx)
     string (REPLACE "-D" "" CPPDEFS "${CPPDEFS}")
     add_compile_definitions ("${CPPDEFS}")
 
-    # Include machine file here
-    include(${CMAKE_SOURCE_DIR}/eamxx/cmake/machine-files/${MACH}.cmake)
+    # Include machine file here. Prefer a compiler-specific file.
+    set(SCREAM_MACH_FILE_ROOT ${CMAKE_SOURCE_DIR}/eamxx/cmake/machine-files)
+    if (EXISTS ${SCREAM_MACH_FILE_ROOT}/${MACH}-${COMPILER}.cmake)
+      include(${SCREAM_MACH_FILE_ROOT}/${MACH}-${COMPILER}.cmake)
+    else()
+      include(${SCREAM_MACH_FILE_ROOT}/${MACH}.cmake)
+    endif()
     add_subdirectory("eamxx")
   endif()
 

components/eam/src/physics/cam/shoc.F90

@@ -2275,7 +2275,6 @@ subroutine shoc_assumed_pdf(&
   real(rtype) epsterm
   real(rtype) sqrtqw2_1, sqrtqw2_2, sqrtthl2_1, sqrtthl2_2
   real(rtype) thl_tol, rt_tol, w_tol_sqd, w_thresh
-  character(len=200) :: err_msg
 
   ! variables on thermo grid
   real(rtype) :: wthl_sec_zt(shcol,nlev)

components/eamxx/CMakeLists.txt

@@ -28,9 +28,6 @@ if (SCREAM_CIME_BUILD)
        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime)
 endif ()
 
-message("Kokkos_ENABLE_CUDA ${Kokkos_ENABLE_CUDA}")
-message("Kokkos_ENABLE_HIP  ${Kokkos_ENABLE_HIP}")
-
 if (Kokkos_ENABLE_CUDA)
   if (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE)
     if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK)
@@ -54,7 +51,12 @@ endif()
 # to be on. For now, simply ensure Kokkos Serial is enabled
 option (Kokkos_ENABLE_SERIAL "" ON)
 
-set (CMAKE_CXX_STANDARD 14)
+# MAM support requires C++17 -- hopefully SCREAM itself will get there soon
+if (SCREAM_ENABLE_MAM)
+  set(CMAKE_CXX_STANDARD 17)
+else()
+  set(CMAKE_CXX_STANDARD 14)
+endif()
 
 if (NOT SCREAM_CIME_BUILD)
   project(SCREAM CXX C Fortran)
@@ -157,9 +159,6 @@ else()
   endif()
 endif ()
 set(DEFAULT_FPMODEL "precise")
-set(DEFAULT_MPIRUN_EXE "mpiexec")
-set(DEFAULT_MPI_EXTRA_ARGS "--bind-to core")
-set(DEFAULT_MPI_NP_FLAG "--map-by")
 set(DEFAULT_LIB_ONLY FALSE)
 if (SCREAM_CIME_BUILD)
   set(DEFAULT_LIB_ONLY TRUE)
@@ -192,6 +191,9 @@ endif()
 
 if (DEFINED ENV{SCREAM_MACHINE})
   set(DEFAULT_SCREAM_MACHINE $ENV{SCREAM_MACHINE})
+elseif(MACH)
+  # MACH comes from CIME build system
+  set(DEFAULT_SCREAM_MACHINE ${MACH})
 endif()
 
 set(DEFAULT_SMALL_KERNELS FALSE)
@@ -203,22 +205,24 @@ endif()
 set(SCREAM_MIMIC_GPU ${DEFAULT_MIMIC_GPU} CACHE BOOL "Mimic GPU to correctness-test inter-column parallelism on non-GPU platform")
 set(SCREAM_PACK_CHECK_BOUNDS FALSE CACHE BOOL "If defined, scream::pack objects check indices against bounds")
 set(SCREAM_TEST_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR}/data CACHE PATH "Location of data files generated by tests")
-set(SCREAM_MPI_ERRORS_ARE_FATAL TRUE CACHE BOOL "Whether MPI errors should abort (default TRUE). If false, errors should be handled.")
-set(SCREAM_MPIRUN_EXE ${DEFAULT_MPIRUN_EXE} CACHE STRING "The executable name for mpirun")
-set(SCREAM_MPI_EXTRA_ARGS ${DEFAULT_MPI_EXTRA_ARGS} CACHE STRING "Options for mpirun")
-set(SCREAM_MPI_NP_FLAG ${DEFAULT_MPI_NP_FLAG} CACHE STRING "The mpirun flag for designating the total number of ranks")
 set(SCREAM_LIB_ONLY ${DEFAULT_LIB_ONLY} CACHE BOOL "Only build libraries, no exes")
 set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to netcdf fortran installation")
 set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation")
 set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine")
 option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON)
+option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF)
 set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels")
 if (NOT SCREAM_SMALL_KERNELS)
   set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "")
 endif()
 
+### The following test only runs on quartz or docker container
+if (NOT DEFINED RUN_ML_CORRECTION_TEST)
+  set(RUN_ML_CORRECTION_TEST FALSE)
+endif()
+
 # Handle input root
-if (SCREAM_MACHINE)
+if (SCREAM_MACHINE AND NOT SCREAM_INPUT_ROOT)
     execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/query-cime ${SCREAM_MACHINE} DIN_LOC_ROOT
     RESULT_VARIABLE QC_STATUS
     OUTPUT_VARIABLE QC_OUTPUT
@@ -428,6 +432,7 @@ if (SCREAM_DOUBLE_PRECISION)
   endif()
 endif()
 
+print_var(SCREAM_MACHINE)
 print_var(EAMXX_ENABLE_GPU)
 print_var(CUDA_BUILD)
 print_var(HIP_BUILD)
@@ -440,7 +445,6 @@ print_var(SCREAM_SMALL_PACK_SIZE)
 print_var(SCREAM_POSSIBLY_NO_PACK_SIZE)
 print_var(SCREAM_LINK_FLAGS)
 print_var(SCREAM_FPMODEL)
-print_var(SCREAM_MPIRUN_EXE)
 print_var(SCREAM_LIB_ONLY)
 print_var(SCREAM_TPL_LIBRARIES)
 print_var(SCREAM_TEST_MAX_THREADS)

components/eamxx/CTestConfig.cmake

@@ -1,5 +1,7 @@
 
-set(CTEST_SITE $ENV{SCREAM_MACHINE})
+if (NOT CTEST_TEST AND DEFINED ENV{SCREAM_MACHINE})
+  set(CTEST_SITE $ENV{SCREAM_MACHINE})
+endif()
 
 set(CTEST_PROJECT_NAME "SCREAM")
 string(TIMESTAMP CURRTIME "%H:%M:%S" UTC)