From 1bb81dfc990905ed5059efb9ecd055a0466c885a Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Fri, 25 Aug 2023 11:43:47 -0400 Subject: [PATCH 01/10] Reads compiler/machine configs from E3SM CIME This merge update the capability of using E3SM configurations including compiler and compiler options. * uses MPICXX from E3SM CIME as a default Omega compiler * User can specify compiler and compiler/linker options using OMEGA_CXX_COMPILER, OMEGA_CXX_FLAGS, and OMEGA_LINK_OPTIONS * adds gen_machine_info.py to read config_machines.xml in E3SM CIME * adds the unit testing for data types * added new cmake variables of OMEGA_INCLUDE_DIRECTORIES OMEGA_LINK_DIRECTORIES * better supports compilers that are not wrapped in a compiler wrapper such as hipcc * uses cmake variables from E3SM for GPU models including CUDA_FLAGS, HIP, and SYCL * supports older Python versions --- components/omega/CMakeLists.txt | 9 ++ components/omega/OmegaBuild.cmake | 136 +++++++++++++--- components/omega/doc/devGuide/CMakeBuild.md | 16 +- components/omega/doc/userGuide/OmegaBuild.md | 86 ++++++---- components/omega/gen_machine_info.py | 159 +++++++++++++++++++ components/omega/src/CMakeLists.txt | 37 ++--- components/omega/src/drivers/DrvDummy.cpp | 1 - components/omega/test/CMakeLists.txt | 55 ++++++- 8 files changed, 424 insertions(+), 75 deletions(-) create mode 100644 components/omega/gen_machine_info.py diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt index 369b52655782..dcecbb33df63 100644 --- a/components/omega/CMakeLists.txt +++ b/components/omega/CMakeLists.txt @@ -5,6 +5,15 @@ # 2) e3sm build +# Included scripts do automatic cmake_policy() PUSH and POP +cmake_policy(SET CMP0011 NEW) + +# list command no longer ignores empty elements +cmake_policy(SET CMP0007 NEW) + +# Only interpret if() arguments as variables or keywords when unquoted +cmake_policy(SET CMP0054 NEW) + ########################################################### # STEP 1: Setup # # # diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 78ecf78f5b9e..9f8fb9520e23 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -13,6 +13,8 @@ set_property(CACHE OMEGA_BUILD_MODE PROPERTY STRINGS ${OMEGA_BUILD_MODES}) set(OMEGA_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}) set(OMEGA_DEFAULT_BUILD_TYPE Release) # Debug or Release +set(E3SM_CIME_ROOT ${OMEGA_SOURCE_DIR}/../../cime) +set(E3SM_CIMECONFIG_ROOT ${OMEGA_SOURCE_DIR}/../../cime_config) ########################### # Public variables # @@ -21,11 +23,22 @@ macro(setup_common_variables) option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF) - if(NOT DEFINED OMEGA_CXX_FLAGS ) - # initialize cxx flags as an empty list + if(NOT DEFINED OMEGA_CXX_FLAGS) set(OMEGA_CXX_FLAGS "") endif() + if(NOT DEFINED OMEGA_INCLUDE_DIRECTORIES) + set(OMEGA_INCLUDE_DIRECTORIES "") + endif() + + if(NOT DEFINED OMEGA_LINK_DIRECTORIES) + set(OMEGA_LINK_DIRECTORIES "") + endif() + + if(NOT DEFINED OMEGA_LINK_OPTIONS) + set(OMEGA_LINK_OPTIONS "") + endif() + endmacro() ########################### @@ -33,14 +46,40 @@ endmacro() ########################### macro(preset) - # set CMAKE_CXX_COMPILER from OMEGA_CXX_COMPILER - if(OMEGA_CXX_COMPILER) - execute_process(COMMAND which ${OMEGA_CXX_COMPILER} - OUTPUT_VARIABLE _OMEGA_CXX_COMPILER - OUTPUT_STRIP_TRAILING_WHITESPACE) - set(CMAKE_CXX_COMPILER ${_OMEGA_CXX_COMPILER}) + find_package (Python COMPONENTS Interpreter) + + if(NOT Python_FOUND) + message(FATAL_ERROR "Python is not available, CMake will exit." ) + endif() + + set(_TMP_CMAKE_FILE ${CMAKE_CURRENT_BINARY_DIR}/_Omega.cmake) + + if(DEFINED OMEGA_CIME_COMPILER) + set(_TMP_COMPILER "-c ${OMEGA_CIME_COMPILER}") + else() + set(_TMP_COMPILER "") endif() + execute_process(COMMAND ${Python_EXECUTABLE} gen_machine_info.py + -p ${E3SM_CIME_ROOT} -o ${_TMP_CMAKE_FILE} ${_TMP_COMPILER} + OUTPUT_QUIET ERROR_QUIET + WORKING_DIRECTORY ${OMEGA_SOURCE_DIR} + OUTPUT_VARIABLE _MACHINE_INFO) + + include(${_TMP_CMAKE_FILE}) + file(REMOVE ${_TMP_CMAKE_FILE}) + + if(NOT OMEGA_CXX_COMPILER) + if (MPILIB STREQUAL "mpi-serial") + set(OMEGA_CXX_COMPILER ${SCXX}) + else() + set(OMEGA_CXX_COMPILER ${MPICXX}) + endif() + endif() + + set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) + message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}") + endmacro() macro(setup_standalone_build) @@ -57,8 +96,6 @@ macro(setup_standalone_build) EXISTS ${OMEGA_SOURCE_DIR}/../../externals) set(E3SM_SOURCE_DIR ${OMEGA_SOURCE_DIR}/../../components) - set(E3SM_CIME_ROOT ${OMEGA_SOURCE_DIR}/../../cime) - set(E3SM_CIMECONFIG_ROOT ${OMEGA_SOURCE_DIR}/../../cime_config) set(E3SM_EXTERNALS_ROOT ${OMEGA_SOURCE_DIR}/../../externals) else() @@ -77,8 +114,6 @@ macro(setup_e3sm_build) setup_common_variables() set(OMEGA_BUILD_TYPE ${E3SM_DEFAULT_BUILD_TYPE}) - set(E3SM_CIME_ROOT ${CIMEROOT}) - set(E3SM_CIMECONFIG_ROOT ${E3SM_SOURCE_DIR}/../cime_config) set(E3SM_EXTERNALS_ROOT ${E3SM_SOURCE_DIR}/../externals) @@ -94,14 +129,76 @@ endmacro() ################################ macro(update_variables) + # Set the build type + set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE}) - if(OMEGA_DEBUG) - add_compile_definitions(OMEGA_DEBUG) - add_compile_definitions(LOG_UNBUFFERED_LOGGING="1") + if (NETCDF_PATH) + list(APPEND OMEGA_INCLUDE_DIRECTORIES "${NETCDF_PATH}/include") + list(APPEND OMEGA_LINK_DIRECTORIES "${NETCDF_PATH}/lib") + list(APPEND OMEGA_LINK_OPTIONS "-lnetcdf") endif() - # Set the build type - set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE}) + if (PNETCDF_PATH) + list(APPEND OMEGA_INCLUDE_DIRECTORIES "${PNETCDF_PATH}/include") + list(APPEND OMEGA_LINK_DIRECTORIES "${PNETCDF_PATH}/lib") + list(APPEND OMEGA_LINK_OPTIONS "-lpnetcdf") + endif() + + string(CONCAT _TestMPISource + "#include \"mpi.h\"\n" + "int main(int argc, char* argv[])\n" + "{MPI_Init(&argc, &argv)\; return 0\;}\n") + set(_TestMPISrcFile ${CMAKE_CURRENT_BINARY_DIR}/_testMPI.cpp) + set(_TestMPIObjFile ${CMAKE_CURRENT_BINARY_DIR}/_testMPI.o) + file(WRITE ${_TestMPISrcFile} ${_TestMPISource}) + + execute_process( + COMMAND ${OMEGA_CXX_COMPILER} -c ${_TestMPISrcFile} -o ${_TestMPIObjFile} + OUTPUT_QUIET ERROR_QUIET + RESULT_VARIABLE _MPI_TEST_RESULT) + + if(OMEGA_BUILD_TYPE EQUAL Release) + file(REMOVE ${_TestMPISrcFile}) + file(REMOVE ${_TestMPIObjFile}) + endif() + + if (NOT _MPI_TEST_RESULT EQUAL 0) + if (_MPI_TEST_RESULT MATCHES "^[-]?[0-9]+$") + find_package(MPI) + if(MPI_CXX_FOUND) + list(APPEND OMEGA_INCLUDE_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}") + list(APPEND OMEGA_LINK_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}/../lib") + list(APPEND OMEGA_LINK_OPTIONS "-lmpi") + else() + message(FATAL_ERROR "MPI is not found" ) + endif() + else() + message(FATAL_ERROR "MPI test failure: ${_MPI_TEST_RESULT}" ) + endif() + endif() + + if(USE_CUDA) + list(APPEND OMEGA_CXX_FLAGS ${CUDA_FLAGS}) + + elseif(USE_HIP) + list(APPEND OMEGA_CXX_FLAGS ${HIP_FLAGS}) + + elseif(USE_SYCL) + list(APPEND OMEGA_CXX_FLAGS ${SYCL_FLAGS}) + + else() + list(APPEND OMEGA_CXX_FLAGS ${CXXFLAGS}) + + if(CXX_LIBS) + list(APPEND OMEGA_LINK_OPTIONS ${CXX_LIBS}) + endif() + + endif() + + message(STATUS "OMEGA_CXX_FLAGS = ${CMAKE_CXX_FLAGS}") + message(STATUS "OMEGA_INCLUDE_DIRECTORIES = ${OMEGA_INCLUDE_DIRECTORIES}") + message(STATUS "OMEGA_LINK_DIRECTORIES = ${OMEGA_LINK_DIRECTORIES}") + message(STATUS "OMEGA_LINK_OPTIONS = ${OMEGA_LINK_OPTIONS}") if(OMEGA_INSTALL_PREFIX) set(CMAKE_INSTALL_PREFIX ${OMEGA_INSTALL_PREFIX}) @@ -122,6 +219,7 @@ macro(update_variables) endif() else() + set(YAKL_ARCH "") endif() @@ -145,10 +243,10 @@ macro(check_setup) #message("OMEGA_BUILD_MODE = ${OMEGA_BUILD_MODE}") if(OMEGA_BUILD_MODE STREQUAL "E3SM") - message("*** Omega E3SM-component Build ***") + message(STATUS "*** Omega E3SM-component Build ***") elseif(${OMEGA_BUILD_MODE} STREQUAL "STANDALONE") - message("*** Omega Standalone Build ***") + message(STATUS "*** Omega Standalone Build ***") else() diff --git a/components/omega/doc/devGuide/CMakeBuild.md b/components/omega/doc/devGuide/CMakeBuild.md index 5739f5738ec0..490aab49520a 100644 --- a/components/omega/doc/devGuide/CMakeBuild.md +++ b/components/omega/doc/devGuide/CMakeBuild.md @@ -14,6 +14,8 @@ steps: Setup, Update, Build, and Output. The build step consists of adding three subdirectories that drive builds for external libraries, the Omega model, and optional tests. +Python is required to use this build system. + ## Step 1: Setup During this step, the build-controlling variables are configured. @@ -45,7 +47,12 @@ OMEGA_SOURCE_DIR: Directory where the top-level Omega CMakeLists.txt is located OMEGA_INSTALL_PREFIX: User-defined output directory for the library and executable OMEGA_ARCH: User-defined programming framework (e.g., "CUDA", "HIP", "OPENMP", "SYCL", "") OMEGA_${YAKL_ARCH}_FLAGS: Framework-specific compiler flags -OMEGA_CXX_FLAGS: C++ compiler flags +OMEGA_CXX_COMPILER: C++ compiler +OMEGA_CIME_COMPILER: E3SM compiler name defined in config_machines.xml +OMEGA_CXX_FLAGS: a list for C++ compiler flags +OMEGA_LINK_OPTIONS: a list for linker flags +OMEGA_INCLUDE_DIRECTORIES: Directory where header files are located +OMEGA_LINK_DIRECTORIES: Directory where library files are located OMEGA_BUILD_EXECUTABLE: Enable building the Omega executable OMEGA_BUILD_TEST: Enable building Omega tests ``` @@ -65,6 +72,8 @@ CMake variables ``` CMAKE_CURRENT_SOURCE_DIR CMAKE_CXX_STANDARD +CMAKE_CXX_COMPILER +CMAKE_CXX_FLAGS CMAKE_CURRENT_LIST_DIR CMAKE_BUILD_TYPE CMAKE_INSTALL_PREFIX @@ -75,8 +84,9 @@ CMAKE_VERSION ## Step 2: Update In this step, CMake is configured, and external library variables, -such as YAKL, are set based on the settings defined in the Setup step. -The integrity of the build setup is verified at the end of this step. +such as YAKL, MPI, NetCDF, and PNetCDF, are set based on the settings +defined in the Setup step. The integrity of the build setup is verified +at the end of this step. ## Step 3: Build diff --git a/components/omega/doc/userGuide/OmegaBuild.md b/components/omega/doc/userGuide/OmegaBuild.md index b3145ab0a68d..4c86b3038757 100644 --- a/components/omega/doc/userGuide/OmegaBuild.md +++ b/components/omega/doc/userGuide/OmegaBuild.md @@ -2,10 +2,18 @@ # CMake-based Omega Build -Omega's build system is built upon the CMake build tool, which provides -a robust foundation for managing the build process. +Omega's build system is constructed using the CMake build tool, +offering a strong foundation for managing the building process. -The Omega build system supports two modes: standalone and E3SM component. +The Omega build system has two modes: standalone and E3SM component. + +At the start of the Omega build, the system reads the E3SM machine file +(`config_machines.xml`) for both standalone and E3SM component builds. +Following this, it configures CMake variables and environment variables +based on the computing system where the build is taking place, as well as +user input from the CMake command-line. + +For the Omega build system to function, a Python interpreter is necessary. ## Standalone Build @@ -15,19 +23,36 @@ The standard practice is for a user to create a separate directory where the build should take place and the commands below should be launched from that directory. -To perform a standalone build, you need to execute the cmake command with -the required CMake and Omega parameters. For example, you can specify the -CC parameter as the C++ compiler and enable the ctest option to guide the -Omega build process. The following example demonstrates building -a standalone Omega with a test suite. +To utilize a compiler name that's defined in the E3SM machine configuration +file (`config_machines.xml`), employ the `OMEGA_CIME_COMPILER` CMake variable, +as illustrated below: + +```sh +>> cmake \ + -DOMEGA_CIME_COMPILER=crayclang \ + ${E3SM_HOME}/components/omega +``` + +To employ a specific compiler, users can utilize the `OMEGA_CXX_COMPILER` +CMake variable, providing CMake with the compiler's path. ```sh >> cmake \ - -DOMEGA_BUILD_TEST=ON \ -DOMEGA_CXX_COMPILER=CC \ ${E3SM_HOME}/components/omega ``` +`OMEGA_CXX_COMPILER` overrides `OMEGA_CIME_COMPILER`. + +To enable the ctest-based unittest option, include the `OMEGA_BUILD_TEST` +option as illustrated below. + +```sh +>> cmake \ + -DOMEGA_BUILD_TEST=ON \ + ${E3SM_HOME}/components/omega +``` + Once the `cmake` command succeeds, the directory where the command was executed will contain the following files and directories. @@ -43,31 +68,34 @@ src test ``` -To build the Omega, execute the `make` command in the directory. +To build the Omega, execute the `make` command in the build directory. + +Typical output will look something like: ```sh >> make Scanning dependencies of target yakl -[ 11%] Building Fortran object external/YAKL/CMakeFiles/yakl.dir/src/YAKL_gator_mod.F90.o -[ 22%] Building CXX object external/YAKL/CMakeFiles/yakl.dir/src/YAKL.cpp.o -[ 33%] Linking CXX static library libyakl.a -[ 33%] Built target yakl -[ 44%] Building CXX object src/CMakeFiles/OmegaLib.dir/ocn/ocndummy.cpp.o -[ 55%] Linking CXX static library libOmegaLib.a -[ 55%] Built target OmegaLib -[ 66%] Building CXX object src/CMakeFiles/omega.exe.dir/drivers/drvdummy.cpp.o -[ 77%] Linking CXX executable omega.exe -[ 77%] Built target omega.exe -[ 88%] Building CXX object test/CMakeFiles/OMEGA_TEST.dir/testdummy.cpp.o -[100%] Linking CXX executable OMEGA_TEST -[100%] Built target OMEGA_TEST +[ 10%] Building Fortran object external/YAKL/CMakeFiles/yakl.dir/src/YAKL_gator_mod.F90.o +[ 20%] Building CXX object external/YAKL/CMakeFiles/yakl.dir/src/YAKL.cpp.o +[ 30%] Linking CXX static library libyakl.a +[ 30%] Built target yakl +[ 40%] Building CXX object src/CMakeFiles/OmegaLib.dir/base/MachEnv.cpp.o +[ 50%] Building CXX object src/CMakeFiles/OmegaLib.dir/ocn/OcnDummy.cpp.o +[ 60%] Linking CXX static library libOmegaLib.a +[ 60%] Built target OmegaLib +[ 70%] Building CXX object src/CMakeFiles/omega.exe.dir/drivers/DrvDummy.cpp.o +[ 80%] Linking CXX executable omega.exe +[ 80%] Built target omega.exe +[ 90%] Building CXX object test/CMakeFiles/testDataTypes.exe.dir/base/DataTypesTest.cpp.o +[100%] Linking CXX executable testDataTypes.exe +[100%] Built target testDataTypes.exe ``` If the build succeeds, the Omega library and executable are created in the `src` sub-directory of the build directory. To specify the output directory for saving the generated output files, -you can include the `OMEGA_INSTALL_PREFIX` variable in the `cmake` command, +a user can include the `OMEGA_INSTALL_PREFIX` variable in the `cmake` command, as shown below: ```sh @@ -78,7 +106,7 @@ cmake \ ... ``` -Afterwards, you can use the `make install` command to copy the Omega library +Afterwards, a user can use the `make install` command to copy the Omega library and executable to the `lib` and `bin` sub-directories under the directory specified by `DOMEGA_INSTALL_PREFIX`. @@ -87,12 +115,12 @@ To run the Omega test suite, execute the `ctest` command. ```sh >> ctest Test project - Start 1: OMEGA_TEST -1/1 Test #1: OMEGA_TEST ....................... Passed 0.01 sec + Start 1: DATA_TYPES_TEST +1/1 Test #1: DATA_TYPES_TEST .................. Passed 0.03 sec 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.01 sec +Total Test time (real) = 0.04 sec ``` ## E3SM Component Build @@ -142,7 +170,7 @@ endfunction(build_omega) Step 3: Create an E3SM Case -At this stage, you can create any E3SM case as usual without +At this stage, a user can create any E3SM case as usual without requiring any specific configuration for Omega. NOTE: In this version, the compiled library file for Omega, diff --git a/components/omega/gen_machine_info.py b/components/omega/gen_machine_info.py new file mode 100644 index 000000000000..1ad415113cee --- /dev/null +++ b/components/omega/gen_machine_info.py @@ -0,0 +1,159 @@ +import os, sys, shutil, argparse, re, subprocess + +# TODO: print basic info on screen and verbose info too + +pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) + +def parse_cmdline(): + + here, progname = os.path.split(__file__) + + parser = argparse.ArgumentParser( + prog=progname, + description='generate machine specific infomation', + epilog='Contact: ') + + parser.add_argument('-p', '--cimepath', default=os.path.realpath(os.path.join( + here, "..", "..", "cime"))) # CIME root path + parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile path + parser.add_argument('-c', '--compiler') # compiler + parser.add_argument('-v', '--verbose') # verbose output + + args = parser.parse_args() + + return (args.cimepath, args.outpath, args.compiler, args.verbose) + +def main(): + + cimepath, outpath, compiler, verbose = parse_cmdline() + + sys.path.insert(0, cimepath) + + from CIME.XML.machines import Machines + + class OmegaMachines(Machines): + + def __init__(self, cimepath, outpath, compiler): + + super(OmegaMachines, self).__init__() + + self.cimepath = cimepath + self.outpath = outpath + + if compiler is None: + self.compiler = self.get_default_compiler().strip() + else: + self.compiler = compiler.strip() + + self.machpath = os.path.join(self.cimepath, "..", "cime_config", "machines") + self.macrospath = os.path.join(self.machpath, "cmake_macros", "Macros.cmake") + + self.machname = self.get_machine_name() + self.machos = self.get_value("OS") + + def get_modules(self, outvar): + + module_system_node = self.get_child("module_system") + module_system_type = self.get(module_system_node, "type") + check = (True if self.get(module_system_node, + "allow_error")=="true" else False) + + if module_system_type != "module": + print("ERROR: '%s' module system is not supported." % module_system_type) + exit(-1) + + out1 = subprocess.check_output("env", shell=True) + env1 = str(out1, 'UTF-8') + + module_nodes = self.get_children( + "modules", root=module_system_node + ) + + modcmd = "module" + + shcmds = [] + + for module_node in module_nodes: + compiler = self.get(module_node, "compiler") + if compiler is None or re.match(compiler, self.compiler): + command_nodes = self.get_children("command", root=module_node) + for command_node in command_nodes: + name = self.get(command_node, "name") + module = self.text(command_node) + if module is None: + shcmds.append("%s %s" % (modcmd, name)) + else: + shcmds.append("%s %s %s" % (modcmd, name, module)) + + shcmds.append("env") + + out2 = subprocess.check_output(";".join(shcmds), shell=True) + env2 = str(out2, 'UTF-8') + + parsed1 = {} + + for (name, value) in pat_envvar.findall(env1): + parsed1[name] = value + + for (name, value) in pat_envvar.findall(env2): + if name in parsed1: + if parsed1[name] != value: + outvar[name] = value + else: + outvar[name] = value + + def get_envs(self, outvar): + + envvar_nodes = self.get_children(root=self.get_child("environment_variables")) + + for envvar_node in envvar_nodes: + compiler = self.get(envvar_node, "compiler") + if compiler is None or re.match(compiler, self.compiler): + env_nodes = self.get_children("env", root=envvar_node) + for env_node in env_nodes: + name = self.get(envnode, "name") + value = self.text(env_node).strip() + if value is None: + outvar[name] = "" + elif value.startswith("$ENV{") and value.endswith("}"): + vname = value[5:-1] + if vname in outvar: + outvar[name] = outvar[vname] + elif vname in os.environ: + outvar[name] = os.environ[vname] + else: + outvar[name] = "" + else: + outvar[name] = value + + def gen_machinfo(self): + + outvar = {} + + self.get_modules(outvar) + self.get_envs(outvar) + self.write_output(outvar) + + def write_output(self, outvar): + + with open(self.outpath, "w") as f: + f.write("message(STATUS \"Reading E3SM machine info\")\n") + + for key, value in outvar.items(): + f.write("set(ENV{%s} \"%s\")\n" % (key, value)) + + f.write("set(MACH %s)\n" % self.machname) + f.write("set(OS %s)\n" % self.machos) + f.write("set(COMPILER %s)\n" % self.compiler) + f.write("set(CASEROOT %s)\n" % os.path.realpath(self.machpath)) + f.write("include(%s)\n" % os.path.realpath(self.macrospath)) + + f.write("message(STATUS \"End of reading E3SM machine info\")\n") + + mach = OmegaMachines(cimepath, outpath, compiler) + + mach.gen_machinfo() + + +if __name__ == "__main__": + main() diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt index fa180f13173f..b1bfe9da9aad 100644 --- a/components/omega/src/CMakeLists.txt +++ b/components/omega/src/CMakeLists.txt @@ -1,26 +1,31 @@ # build Omega # Add source files for the library -file(GLOB _LIBSRC_FILES ocn/*.cpp infra/*.cpp) - -list(JOIN OMEGA_CXX_FLAGS " " _CXX_FLAGS_EXPANDED) +file(GLOB _LIBSRC_FILES base/*.cpp infra/*.cpp ocn/*.cpp) # Create the library target add_library(${OMEGA_LIB_NAME} ${_LIBSRC_FILES}) -set_target_properties( - ${OMEGA_LIB_NAME} - PROPERTIES COMPILE_FLAGS "${_CXX_FLAGS_EXPANDED}" -) - -# add link directories -target_link_libraries(${OMEGA_LIB_NAME} spdlog "${OMEGA_LDFLAGS}") +target_link_libraries(${OMEGA_LIB_NAME} spdlog "${OMEGA_LINK_FLAGS}") # add include directories target_include_directories( - ${OMEGA_LIB_NAME} - PRIVATE - "${OMEGA_SOURCE_DIR}/src/base" - "${OMEGA_SOURCE_DIR}/src/infra" + ${OMEGA_LIB_NAME} + PRIVATE + "${OMEGA_SOURCE_DIR}/src/base" + "${OMEGA_SOURCE_DIR}/src/infra" + ${OMEGA_INCLUDE_DIRECTORIES} +) + +target_link_directories( + ${OMEGA_LIB_NAME} + PRIVATE + ${OMEGA_LINK_DIRECTORIES} +) + +target_link_options( + ${OMEGA_LIB_NAME} + PRIVATE + ${OMEGA_LINK_OPTIONS} ) # include yakl cmake utility @@ -44,10 +49,6 @@ if(OMEGA_BUILD_EXECUTABLE) ) # Create the executable target add_executable(${OMEGA_EXE_NAME} ${EXESRC_FILES}) - set_target_properties( - ${OMEGA_EXE_NAME} - PROPERTIES COMPILE_FLAGS "${_CXX_FLAGS_EXPANDED}" - ) target_include_directories( ${OMEGA_EXE_NAME} PRIVATE diff --git a/components/omega/src/drivers/DrvDummy.cpp b/components/omega/src/drivers/DrvDummy.cpp index a3b2be802421..e66d9f7cc6d0 100644 --- a/components/omega/src/drivers/DrvDummy.cpp +++ b/components/omega/src/drivers/DrvDummy.cpp @@ -1,6 +1,5 @@ // OCN dummy driver -#include "Logging.h" #include void dummy(int argc, char **argv); diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index db64ccede091..db2469f193ba 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -1,14 +1,59 @@ +# Omega Unit Tests -# Add tests +# Add data type test -add_executable(testLogging.exe infra/LoggingTest.cpp) -target_link_libraries(testLogging.exe ${OMEGA_LIB_NAME}) +set(_TestDataTypesName testDataTypes.exe) + +add_executable(${_TestDataTypesName} base/DataTypesTest.cpp) +target_link_libraries(${_TestDataTypesName} ${OMEGA_LIB_NAME}) +target_include_directories( + ${_TestDataTypesName} + PRIVATE + "${OMEGA_SOURCE_DIR}/src/base" + ${OMEGA_INCLUDE_DIRECTORIES} +) + +target_link_directories( + ${_TestDataTypesName} + PRIVATE + ${OMEGA_LINK_DIRECTORIES} +) + +target_link_options( + ${_TestDataTypesName} + PRIVATE + ${OMEGA_LINK_OPTIONS} +) + +add_test(NAME DATA_TYPES_TEST COMMAND ${_TestDataTypesName}) + +# Add machine env test +# NOTE: disabled temporary due to not-launching mininum 8 ranks +#add_executable(testMachEnv.exe base/MachEnvTest.cpp) +#target_link_libraries(testMachEnv.exe ${OMEGA_LIB_NAME}) +#target_include_directories( +# testMachEnv.exe +# PRIVATE +# "${OMEGA_SOURCE_DIR}/src/base" +#) +#add_test(NAME MACHINE_ENV_TEST COMMAND testMachEnv.exe) + +set(_TestLoggingName testLogging.exe) + +add_executable(${_TestLoggingName} infra/LoggingTest.cpp) +target_link_libraries(${_TestLoggingName} ${OMEGA_LIB_NAME}) target_include_directories( - testLogging.exe + ${_TestLoggingName} PRIVATE "${OMEGA_SOURCE_DIR}/src/base" "${OMEGA_SOURCE_DIR}/src/infra" ) + add_test(NAME LOGGING_TEST COMMAND testLogging.exe) -set_tests_properties(LOGGING_TEST + +# set test properties +set_tests_properties( + DATA_TYPES_TEST +# MACHINE_ENV_TEST + LOGGING_TEST PROPERTIES FAIL_REGULAR_EXPRESSION "FAIL") From 3c3852295a6b3fa2aa7bedb493627ab1abbb49cf Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 20 Sep 2023 14:21:19 -0400 Subject: [PATCH 02/10] fixing linting issues1 --- components/omega/gen_machine_info.py | 31 ++++++++++++++++------------ components/omega/src/CMakeLists.txt | 6 +++--- components/omega/test/CMakeLists.txt | 6 +++--- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/components/omega/gen_machine_info.py b/components/omega/gen_machine_info.py index 1ad415113cee..0cf1fc8812fd 100644 --- a/components/omega/gen_machine_info.py +++ b/components/omega/gen_machine_info.py @@ -1,4 +1,9 @@ -import os, sys, shutil, argparse, re, subprocess +import argparse +import os +import re +import shutil +import subprocess +import sys # TODO: print basic info on screen and verbose info too @@ -20,7 +25,7 @@ def parse_cmdline(): parser.add_argument('-v', '--verbose') # verbose output args = parser.parse_args() - + return (args.cimepath, args.outpath, args.compiler, args.verbose) def main(): @@ -39,7 +44,7 @@ def __init__(self, cimepath, outpath, compiler): self.cimepath = cimepath self.outpath = outpath - + if compiler is None: self.compiler = self.get_default_compiler().strip() else: @@ -53,13 +58,13 @@ def __init__(self, cimepath, outpath, compiler): def get_modules(self, outvar): - module_system_node = self.get_child("module_system") + module_system_node = self.get_child("module_system") module_system_type = self.get(module_system_node, "type") check = (True if self.get(module_system_node, "allow_error")=="true" else False) if module_system_type != "module": - print("ERROR: '%s' module system is not supported." % module_system_type) + print(f"ERROR: '{module_system_type}' module system is not supported.") exit(-1) out1 = subprocess.check_output("env", shell=True) @@ -81,9 +86,9 @@ def get_modules(self, outvar): name = self.get(command_node, "name") module = self.text(command_node) if module is None: - shcmds.append("%s %s" % (modcmd, name)) + shcmds.append(f"{modcmd} {name}") else: - shcmds.append("%s %s %s" % (modcmd, name, module)) + shcmds.append(f"{modcmd} {name} {module}") shcmds.append("env") @@ -141,12 +146,12 @@ def write_output(self, outvar): for key, value in outvar.items(): f.write("set(ENV{%s} \"%s\")\n" % (key, value)) - - f.write("set(MACH %s)\n" % self.machname) - f.write("set(OS %s)\n" % self.machos) - f.write("set(COMPILER %s)\n" % self.compiler) - f.write("set(CASEROOT %s)\n" % os.path.realpath(self.machpath)) - f.write("include(%s)\n" % os.path.realpath(self.macrospath)) + + f.write(f"set(MACH {self.machname})\n") + f.write(f"set(OS {self.machos})\n") + f.write(f"set(COMPILER {self.compiler})\n") + f.write(f"set(CASEROOT {os.path.realpath(self.machpath)})\n") + f.write(f"include({os.path.realpath(self.macrospath)})\n") f.write("message(STATUS \"End of reading E3SM machine info\")\n") diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt index b1bfe9da9aad..d50347312ba6 100644 --- a/components/omega/src/CMakeLists.txt +++ b/components/omega/src/CMakeLists.txt @@ -10,7 +10,7 @@ target_link_libraries(${OMEGA_LIB_NAME} spdlog "${OMEGA_LINK_FLAGS}") # add include directories target_include_directories( ${OMEGA_LIB_NAME} - PRIVATE + PRIVATE "${OMEGA_SOURCE_DIR}/src/base" "${OMEGA_SOURCE_DIR}/src/infra" ${OMEGA_INCLUDE_DIRECTORIES} @@ -18,13 +18,13 @@ target_include_directories( target_link_directories( ${OMEGA_LIB_NAME} - PRIVATE + PRIVATE ${OMEGA_LINK_DIRECTORIES} ) target_link_options( ${OMEGA_LIB_NAME} - PRIVATE + PRIVATE ${OMEGA_LINK_OPTIONS} ) diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index db2469f193ba..c5bcca4b78ac 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -15,13 +15,13 @@ target_include_directories( target_link_directories( ${_TestDataTypesName} - PRIVATE + PRIVATE ${OMEGA_LINK_DIRECTORIES} ) target_link_options( ${_TestDataTypesName} - PRIVATE + PRIVATE ${OMEGA_LINK_OPTIONS} ) @@ -55,5 +55,5 @@ add_test(NAME LOGGING_TEST COMMAND testLogging.exe) set_tests_properties( DATA_TYPES_TEST # MACHINE_ENV_TEST - LOGGING_TEST + LOGGING_TEST PROPERTIES FAIL_REGULAR_EXPRESSION "FAIL") From f5856f52faaec107153b9e45f143b32cdfe1e541 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Thu, 21 Sep 2023 11:01:24 -0400 Subject: [PATCH 03/10] gets MPI_EXEC from CIME; updated gen_machine_info.py to pass linting tests --- components/omega/OmegaBuild.cmake | 1 + components/omega/gen_machine_info.py | 240 +++++++++++++++------------ 2 files changed, 136 insertions(+), 105 deletions(-) diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 9f8fb9520e23..f9dbfea582e9 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -79,6 +79,7 @@ macro(preset) set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}") + message(STATUS "MPI_EXEC = ${MPI_EXEC}") endmacro() diff --git a/components/omega/gen_machine_info.py b/components/omega/gen_machine_info.py index 0cf1fc8812fd..9892384c0506 100644 --- a/components/omega/gen_machine_info.py +++ b/components/omega/gen_machine_info.py @@ -1,162 +1,192 @@ import argparse import os import re -import shutil import subprocess import sys - -# TODO: print basic info on screen and verbose info too +import typing pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) + def parse_cmdline(): here, progname = os.path.split(__file__) parser = argparse.ArgumentParser( - prog=progname, - description='generate machine specific infomation', - epilog='Contact: ') + prog=progname, + description='generate machine specific infomation', + epilog='Contact: ') - parser.add_argument('-p', '--cimepath', default=os.path.realpath(os.path.join( - here, "..", "..", "cime"))) # CIME root path - parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile path - parser.add_argument('-c', '--compiler') # compiler - parser.add_argument('-v', '--verbose') # verbose output + parser.add_argument('-p', '--cimepath', default=os.path.realpath( + os.path.join(here, "..", "..", "cime"))) # CIME root + parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile + parser.add_argument('-c', '--compiler') # compiler + parser.add_argument('-v', '--verbose') # verbose output - args = parser.parse_args() + # cimepath, outpath, compiler, verbose + return parser.parse_args() - return (args.cimepath, args.outpath, args.compiler, args.verbose) -def main(): +args = parse_cmdline() - cimepath, outpath, compiler, verbose = parse_cmdline() +sys.path.insert(0, args.cimepath) +from CIME.XML.machines import Machines # noqa: E402 - sys.path.insert(0, cimepath) +sys.path.pop(0) - from CIME.XML.machines import Machines - class OmegaMachines(Machines): +class OmegaMachines(Machines): - def __init__(self, cimepath, outpath, compiler): + def __init__(self, cimepath, outpath, compiler): - super(OmegaMachines, self).__init__() + super(OmegaMachines, self).__init__() - self.cimepath = cimepath - self.outpath = outpath + self.cimepath = cimepath + self.outpath = outpath + self.mpiexec = "" - if compiler is None: - self.compiler = self.get_default_compiler().strip() - else: - self.compiler = compiler.strip() + if compiler is None: + self.compiler = self.get_default_compiler().strip() + else: + self.compiler = compiler.strip() - self.machpath = os.path.join(self.cimepath, "..", "cime_config", "machines") - self.macrospath = os.path.join(self.machpath, "cmake_macros", "Macros.cmake") + self.machpath = os.path.join(self.cimepath, "..", "cime_config", + "machines") + self.macrospath = os.path.join(self.machpath, "cmake_macros", + "Macros.cmake") - self.machname = self.get_machine_name() - self.machos = self.get_value("OS") + self.machname = self.get_machine_name() + self.machos = self.get_value("OS") - def get_modules(self, outvar): + def get_modules(self, outvar): - module_system_node = self.get_child("module_system") - module_system_type = self.get(module_system_node, "type") - check = (True if self.get(module_system_node, - "allow_error")=="true" else False) + module_system_node = self.get_child("module_system") + module_system_type = self.get(module_system_node, "type") - if module_system_type != "module": - print(f"ERROR: '{module_system_type}' module system is not supported.") - exit(-1) + # check = (True if self.get(module_system_node, + # "allow_error") == "true" else False) - out1 = subprocess.check_output("env", shell=True) - env1 = str(out1, 'UTF-8') + if module_system_type != "module": + print((f"ERROR: '{module_system_type}' " + "module system is not supported.")) + exit(-1) - module_nodes = self.get_children( - "modules", root=module_system_node - ) + out1 = subprocess.check_output("env", shell=True) + env1 = str(out1, 'UTF-8') - modcmd = "module" + module_nodes = self.get_children( + "modules", root=module_system_node + ) - shcmds = [] + modcmd = "module" - for module_node in module_nodes: - compiler = self.get(module_node, "compiler") - if compiler is None or re.match(compiler, self.compiler): - command_nodes = self.get_children("command", root=module_node) - for command_node in command_nodes: - name = self.get(command_node, "name") - module = self.text(command_node) - if module is None: - shcmds.append(f"{modcmd} {name}") - else: - shcmds.append(f"{modcmd} {name} {module}") + shcmds = [] - shcmds.append("env") + for module_node in module_nodes: + compiler = self.get(module_node, "compiler") + if compiler is None or re.match(compiler, self.compiler): + command_nodes = self.get_children("command", + root=module_node) + for command_node in command_nodes: + name = self.get(command_node, "name") + module = self.text(command_node) + if module is None: + shcmds.append(f"{modcmd} {name}") + else: + shcmds.append(f"{modcmd} {name} {module}") - out2 = subprocess.check_output(";".join(shcmds), shell=True) - env2 = str(out2, 'UTF-8') + shcmds.append("env") - parsed1 = {} + out2 = subprocess.check_output(";".join(shcmds), shell=True) + env2 = str(out2, 'UTF-8') - for (name, value) in pat_envvar.findall(env1): - parsed1[name] = value + parsed1 = {} - for (name, value) in pat_envvar.findall(env2): - if name in parsed1: - if parsed1[name] != value: - outvar[name] = value - else: + for (name, value) in pat_envvar.findall(env1): + parsed1[name] = value + + for (name, value) in pat_envvar.findall(env2): + if name in parsed1: + if parsed1[name] != value: outvar[name] = value + else: + outvar[name] = value + + def get_envs(self, outvar): + + envvar_nodes = self.get_children( + root=self.get_child("environment_variables")) + + for envvar_node in envvar_nodes: + compiler = self.get(envvar_node, "compiler") + if compiler is None or re.match(compiler, self.compiler): + env_nodes = self.get_children("env", root=envvar_node) + for env_node in env_nodes: + name = self.get(env_node, "name") + value = self.text(env_node).strip() + if value is None: + outvar[name] = "" + elif value.startswith("$ENV{") and value.endswith("}"): + vname = value[5:-1] + if vname in outvar: + outvar[name] = outvar[vname] + elif vname in os.environ: + outvar[name] = os.environ[vname] + else: + outvar[name] = "" + else: + outvar[name] = value - def get_envs(self, outvar): + def get_mpirun(self, outvar): - envvar_nodes = self.get_children(root=self.get_child("environment_variables")) + mpirun_node = self.get_child("mpirun") + mpirun_mpilib = self.get(mpirun_node, "mpilib") - for envvar_node in envvar_nodes: - compiler = self.get(envvar_node, "compiler") - if compiler is None or re.match(compiler, self.compiler): - env_nodes = self.get_children("env", root=envvar_node) - for env_node in env_nodes: - name = self.get(envnode, "name") - value = self.text(env_node).strip() - if value is None: - outvar[name] = "" - elif value.startswith("$ENV{") and value.endswith("}"): - vname = value[5:-1] - if vname in outvar: - outvar[name] = outvar[vname] - elif vname in os.environ: - outvar[name] = os.environ[vname] - else: - outvar[name] = "" - else: - outvar[name] = value + if mpirun_mpilib != "default": + print(f"ERROR: '{mpirun_mpilib}' mpilib is not supported.") + exit(-1) + + exec_nodes = self.get_children( + "executable", root=mpirun_node + ) - def gen_machinfo(self): + if len(exec_nodes) != 1: + print("ERROR: 'more than one executable nodes in mpirun node") + exit(-1) - outvar = {} + self.mpiexec = self.text(exec_nodes[0]) - self.get_modules(outvar) - self.get_envs(outvar) - self.write_output(outvar) + def gen_machinfo(self): - def write_output(self, outvar): + outvar: typing.Dict[str, int] = {} - with open(self.outpath, "w") as f: - f.write("message(STATUS \"Reading E3SM machine info\")\n") + self.get_modules(outvar) + self.get_envs(outvar) + self.get_mpirun(outvar) + self.write_output(outvar) - for key, value in outvar.items(): - f.write("set(ENV{%s} \"%s\")\n" % (key, value)) + def write_output(self, outvar): - f.write(f"set(MACH {self.machname})\n") - f.write(f"set(OS {self.machos})\n") - f.write(f"set(COMPILER {self.compiler})\n") - f.write(f"set(CASEROOT {os.path.realpath(self.machpath)})\n") - f.write(f"include({os.path.realpath(self.macrospath)})\n") + with open(self.outpath, "w") as f: + f.write("message(STATUS \"Reading E3SM machine info\")\n") - f.write("message(STATUS \"End of reading E3SM machine info\")\n") + for key, value in outvar.items(): + f.write("set(ENV{%s} \"%s\")\n" % (key, value)) - mach = OmegaMachines(cimepath, outpath, compiler) + f.write(f"set(MACH {self.machname})\n") + f.write(f"set(OS {self.machos})\n") + f.write(f"set(COMPILER {self.compiler})\n") + f.write(f"set(MPI_EXEC {self.mpiexec})\n") + f.write(f"set(CASEROOT {self.machpath})\n") + f.write(f"include({self.macrospath})\n") + + f.write(("message(STATUS \"End of reading E3SM " + "machine info\")\n")) + + +def main(): + mach = OmegaMachines(args.cimepath, args.outpath, args.compiler) mach.gen_machinfo() From bace430a356be01d60db2c127ff4c643705be732 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sat, 7 Oct 2023 15:29:06 -0400 Subject: [PATCH 04/10] revised to correclty build Omega on Perlmutter and Frontier. * creates scripts that prerun module commands in the build direcotry: omega_build.sh, omega_run.sh and omega_ctest.sh * YAKL library can be automatically built using HIP or CUDA * updates documents for cmake-based build. --- cime_config/machines/config_machines.xml | 6 +- components/omega/CMakeLists.txt | 5 +- components/omega/OmegaBuild.cmake | 168 +++++---- components/omega/create_scripts.py | 359 +++++++++++++++++++ components/omega/doc/devGuide/CMakeBuild.md | 7 +- components/omega/doc/userGuide/OmegaBuild.md | 26 ++ components/omega/external/CMakeLists.txt | 15 +- components/omega/gen_machine_info.py | 194 ---------- components/omega/src/CMakeLists.txt | 50 +-- components/omega/test/CMakeLists.txt | 137 +++++-- 10 files changed, 622 insertions(+), 345 deletions(-) create mode 100644 components/omega/create_scripts.py delete mode 100644 components/omega/gen_machine_info.py diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 666d7ad993c7..dbf3ff348ead 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4266,13 +4266,15 @@ gcc/9.1.0 - cuda/10.1.243 + + cuda/11.0.3 cuda/11.0.3 - cuda/10.1.243 + + cuda/11.0.3 spectrum-mpi/10.4.0.3-20210112 diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt index dcecbb33df63..22c0f874f8f2 100644 --- a/components/omega/CMakeLists.txt +++ b/components/omega/CMakeLists.txt @@ -28,9 +28,10 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/OmegaBuild.cmake) if (NOT DEFINED PROJECT_NAME) # enter standalone build - preset() + cmake_minimum_required(VERSION 3.21) # the minimum version for HIP support - cmake_minimum_required(VERSION 3.16) # used in E3SM + # Collect machine and compiler info from CIME + preset() project(${OMEGA_PROJECT_NAME} LANGUAGES CXX diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index f9dbfea582e9..f530dffb0f6c 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -1,5 +1,5 @@ ########################### -# Internal variables # +# Build Control Variables # ########################### set(OMEGA_PROJECT_NAME "OmegaOceanModel") @@ -17,8 +17,10 @@ set(E3SM_CIME_ROOT ${OMEGA_SOURCE_DIR}/../../cime) set(E3SM_CIMECONFIG_ROOT ${OMEGA_SOURCE_DIR}/../../cime_config) ########################### -# Public variables # +# Macros # ########################### + +# set build control variables used for both e3sm build and standalone build macro(setup_common_variables) option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF) @@ -27,23 +29,13 @@ macro(setup_common_variables) set(OMEGA_CXX_FLAGS "") endif() - if(NOT DEFINED OMEGA_INCLUDE_DIRECTORIES) - set(OMEGA_INCLUDE_DIRECTORIES "") - endif() - - if(NOT DEFINED OMEGA_LINK_DIRECTORIES) - set(OMEGA_LINK_DIRECTORIES "") - endif() - if(NOT DEFINED OMEGA_LINK_OPTIONS) set(OMEGA_LINK_OPTIONS "") endif() endmacro() -########################### -# Preset Standalone build # -########################### +# Collect machine and compiler info from CIME macro(preset) find_package (Python COMPONENTS Interpreter) @@ -53,36 +45,49 @@ macro(preset) endif() set(_TMP_CMAKE_FILE ${CMAKE_CURRENT_BINARY_DIR}/_Omega.cmake) + set(_PY_OPTS "-p;${E3SM_CIME_ROOT};-o;${_TMP_CMAKE_FILE}") if(DEFINED OMEGA_CIME_COMPILER) - set(_TMP_COMPILER "-c ${OMEGA_CIME_COMPILER}") - else() - set(_TMP_COMPILER "") + list(APPEND _PY_OPTS "-c" "${OMEGA_CIME_COMPILER}") + endif() + + if(DEFINED OMEGA_CIME_MACHINE) + list(APPEND _PY_OPTS "-m" "${OMEGA_CIME_MACHINE}") endif() - execute_process(COMMAND ${Python_EXECUTABLE} gen_machine_info.py - -p ${E3SM_CIME_ROOT} -o ${_TMP_CMAKE_FILE} ${_TMP_COMPILER} + if(OMEGA_BUILD_TYPE STREQUAL "Debug") + list(APPEND _PY_OPTS "-d") + endif() + + execute_process(COMMAND ${Python_EXECUTABLE} create_scripts.py ${_PY_OPTS} OUTPUT_QUIET ERROR_QUIET WORKING_DIRECTORY ${OMEGA_SOURCE_DIR} OUTPUT_VARIABLE _MACHINE_INFO) include(${_TMP_CMAKE_FILE}) - file(REMOVE ${_TMP_CMAKE_FILE}) + if(OMEGA_BUILD_TYPE STREQUAL "Release") + file(REMOVE ${_TMP_CMAKE_FILE}) + endif() - if(NOT OMEGA_CXX_COMPILER) + if(OMEGA_CXX_COMPILER) + find_program(_OMEGA_CXX_COMPILER ${OMEGA_CXX_COMPILER}) + else() if (MPILIB STREQUAL "mpi-serial") - set(OMEGA_CXX_COMPILER ${SCXX}) + find_program(_OMEGA_CXX_COMPILER ${SCXX}) else() - set(OMEGA_CXX_COMPILER ${MPICXX}) + find_program(_OMEGA_CXX_COMPILER ${MPICXX}) endif() endif() + set(OMEGA_CXX_COMPILER ${_OMEGA_CXX_COMPILER}) set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) + message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}") message(STATUS "MPI_EXEC = ${MPI_EXEC}") endmacro() +# set build-control-variables for standalone build macro(setup_standalone_build) setup_common_variables() @@ -110,6 +115,7 @@ macro(setup_standalone_build) endmacro() +# set build-control-variables for e3sm build macro(setup_e3sm_build) setup_common_variables() @@ -117,11 +123,14 @@ macro(setup_e3sm_build) set(OMEGA_BUILD_TYPE ${E3SM_DEFAULT_BUILD_TYPE}) set(E3SM_EXTERNALS_ROOT ${E3SM_SOURCE_DIR}/../externals) + set(OMEGA_CXX_COMPILER ${CMAKE_CXX_COMPILER}) #TODO: set OMEGA_ARCH according to E3SM variables set(OMEGA_ARCH "NOT_DEFINED") set(OMEGA_BUILD_MODE "E3SM") + message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}") + endmacro() @@ -133,18 +142,23 @@ macro(update_variables) # Set the build type set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE}) - if (NETCDF_PATH) - list(APPEND OMEGA_INCLUDE_DIRECTORIES "${NETCDF_PATH}/include") - list(APPEND OMEGA_LINK_DIRECTORIES "${NETCDF_PATH}/lib") - list(APPEND OMEGA_LINK_OPTIONS "-lnetcdf") + # Set compiler and linker flags + if (CXXFLAGS) + separate_arguments(_CXXFLAGS NATIVE_COMMAND ${CXXFLAGS}) + list(APPEND OMEGA_CXX_FLAGS ${_CXXFLAGS}) + endif() + + if (LDFLAGS) + separate_arguments(_LDFLAGS NATIVE_COMMAND ${LDFLAGS}) + list(APPEND OMEGA_LINK_OPTIONS ${_LDFLAGS}) endif() - if (PNETCDF_PATH) - list(APPEND OMEGA_INCLUDE_DIRECTORIES "${PNETCDF_PATH}/include") - list(APPEND OMEGA_LINK_DIRECTORIES "${PNETCDF_PATH}/lib") - list(APPEND OMEGA_LINK_OPTIONS "-lpnetcdf") + if (SLIBS) + separate_arguments(_SLIBS NATIVE_COMMAND ${SLIBS}) + list(APPEND OMEGA_LINK_OPTIONS ${_SLIBS}) endif() + # check if MPI is supported string(CONCAT _TestMPISource "#include \"mpi.h\"\n" "int main(int argc, char* argv[])\n" @@ -156,7 +170,9 @@ macro(update_variables) execute_process( COMMAND ${OMEGA_CXX_COMPILER} -c ${_TestMPISrcFile} -o ${_TestMPIObjFile} OUTPUT_QUIET ERROR_QUIET - RESULT_VARIABLE _MPI_TEST_RESULT) + RESULT_VARIABLE _MPI_TEST_RESULT + OUTPUT_VARIABLE _MPI_TEST_OUTPUT + ERROR_VARIABLE _MPI_TEST_ERROR) if(OMEGA_BUILD_TYPE EQUAL Release) file(REMOVE ${_TestMPISrcFile}) @@ -167,9 +183,10 @@ macro(update_variables) if (_MPI_TEST_RESULT MATCHES "^[-]?[0-9]+$") find_package(MPI) if(MPI_CXX_FOUND) - list(APPEND OMEGA_INCLUDE_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}") - list(APPEND OMEGA_LINK_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}/../lib") - list(APPEND OMEGA_LINK_OPTIONS "-lmpi") + list(APPEND OMEGA_CXX_FLAGS "-I${MPI_CXX_INCLUDE_DIRS}") + list(APPEND OMEGA_LINK_OPTIONS + "-L${MPI_CXX_INCLUDE_DIRS}/../lib" "-lmpi" + ) else() message(FATAL_ERROR "MPI is not found" ) endif() @@ -178,50 +195,67 @@ macro(update_variables) endif() endif() - if(USE_CUDA) - list(APPEND OMEGA_CXX_FLAGS ${CUDA_FLAGS}) + message(STATUS "OMEGA_CXX_FLAGS = ${OMEGA_CXX_FLAGS}") + message(STATUS "OMEGA_LINK_OPTIONS = ${OMEGA_LINK_OPTIONS}") - elseif(USE_HIP) - list(APPEND OMEGA_CXX_FLAGS ${HIP_FLAGS}) + if(OMEGA_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX ${OMEGA_INSTALL_PREFIX}) + endif() - elseif(USE_SYCL) - list(APPEND OMEGA_CXX_FLAGS ${SYCL_FLAGS}) + # Check if CUDA or HIP is supported + if((NOT DEFINED OMEGA_ARCH) OR (OMEGA_ARCH STREQUAL "NOT_DEFINED")) - else() - list(APPEND OMEGA_CXX_FLAGS ${CXXFLAGS}) + execute_process( + COMMAND ${OMEGA_CXX_COMPILER} --version + RESULT_VARIABLE _CXX_VER_RESULT + OUTPUT_VARIABLE _CXX_VER_OUTPUT) - if(CXX_LIBS) - list(APPEND OMEGA_LINK_OPTIONS ${CXX_LIBS}) - endif() + if (_CXX_VER_RESULT EQUAL 0) - endif() + string(REGEX MATCH "HIP|hip" _HIP_CHECK "${_CXX_VER_OUTPUT}") + string(REGEX MATCH "AMD|amd" _AMD_CHECK "${_CXX_VER_OUTPUT}") + string(REGEX MATCH "NVCC|nvcc" _NVCC_CHECK "${_CXX_VER_OUTPUT}") + string(REGEX MATCH "NVIDIA|nvidia" _NVIDIA_CHECK "${_CXX_VER_OUTPUT}") - message(STATUS "OMEGA_CXX_FLAGS = ${CMAKE_CXX_FLAGS}") - message(STATUS "OMEGA_INCLUDE_DIRECTORIES = ${OMEGA_INCLUDE_DIRECTORIES}") - message(STATUS "OMEGA_LINK_DIRECTORIES = ${OMEGA_LINK_DIRECTORIES}") - message(STATUS "OMEGA_LINK_OPTIONS = ${OMEGA_LINK_OPTIONS}") + if(_HIP_CHECK AND _AMD_CHECK) + set(OMEGA_ARCH "HIP") - if(OMEGA_INSTALL_PREFIX) - set(CMAKE_INSTALL_PREFIX ${OMEGA_INSTALL_PREFIX}) + elseif(_NVCC_CHECK AND _NVIDIA_CHECK) + set(OMEGA_ARCH "CUDA") + + else() + set(OMEGA_ARCH "") + + endif() + else() + set(OMEGA_ARCH "") + + endif() endif() - if(DEFINED OMEGA_ARCH) + if(OMEGA_ARCH STREQUAL "CUDA") + enable_language(CUDA) - if(OMEGA_ARCH STREQUAL "NOT_DEFINED") - set(YAKL_ARCH "") + elseif(OMEGA_ARCH STREQUAL "HIP") + enable_language(HIP) - else() - set(YAKL_ARCH ${OMEGA_ARCH}) + elseif(USE_CUDA) + set(OMEGA_ARCH "CUDA") + enable_language(CUDA) - if(OMEGA_${YAKL_ARCH}_FLAGS) - set(YAKL_${YAKL_ARCH}_FLAGS ${OMEGA_${YAKL_ARCH}_FLAGS}) - endif() + elseif(USE_HIP) + set(OMEGA_ARCH "HIP") + enable_language(HIP) - endif() + endif() - else() + set(YAKL_ARCH "${OMEGA_ARCH}") + + if(YAKL_ARCH) - set(YAKL_ARCH "") + if(OMEGA_${YAKL_ARCH}_FLAGS) + set(YAKL_${YAKL_ARCH}_FLAGS ${OMEGA_${YAKL_ARCH}_FLAGS}) + endif() endif() @@ -269,10 +303,14 @@ macro(wrap_outputs) if(OMEGA_INSTALL_PREFIX) - install(TARGETS ${OMEGA_LIB_NAME} LIBRARY DESTINATION "${OMEGA_INSTALL_PREFIX}/lib") + install(TARGETS ${OMEGA_LIB_NAME} + LIBRARY DESTINATION "${OMEGA_INSTALL_PREFIX}/lib" + ) if(OMEGA_BUILD_EXECUTABLE) - install(TARGETS ${OMEGA_EXE_NAME} RUNTIME DESTINATION "${OMEGA_INSTALL_PREFIX}/bin") + install(TARGETS ${OMEGA_EXE_NAME} + RUNTIME DESTINATION "${OMEGA_INSTALL_PREFIX}/bin" + ) endif() endif() diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py new file mode 100644 index 000000000000..1efe1838ebd8 --- /dev/null +++ b/components/omega/create_scripts.py @@ -0,0 +1,359 @@ +""" Collect compiler and machine info from CIME + +This script should be invoked by Omega cmake build system +""" + +import argparse +import os +import re +import subprocess +import sys +import typing +import stat + +pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) + + +def parse_cmdline(): + + here, progname = os.path.split(__file__) + + parser = argparse.ArgumentParser( + prog=progname, + description='generate machine specific infomation', + epilog='Contact: ') + + parser.add_argument('-p', '--cimepath', default=os.path.realpath( + os.path.join(here, "..", "..", "cime"))) # CIME root + parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile + parser.add_argument('-m', '--machine') # machine + parser.add_argument('-c', '--compiler') # compiler + parser.add_argument('-d', '--debug', action='store_const', + const="TRUE", default="FALSE") # debug mode + parser.add_argument('-v', '--verbose') # verbose output + + return parser.parse_args() + +args = parse_cmdline() + +#import Machines class from CIME +sys.path.insert(0, args.cimepath) +from CIME.XML.machines import Machines # noqa: E402 + +sys.path.pop(0) + + +#run a shell command +def run_cmd_no_fail(cmd): + out = subprocess.run(cmd, shell=True, capture_output=True) + return str(out.stdout, 'UTF-8') + + +#main class that extends Machines class +class OmegaMachines(Machines): + + def __init__(self, cimepath, outpath, machine, compiler, debug): + + if isinstance(machine, str): + machine = machine.strip() + + super(OmegaMachines, self).__init__(machine=machine) + + self.cimepath = cimepath + self.outpath = outpath + self.debug = debug + self.mpilib = "" + self.mpiexec = "" + + if compiler is None: + self.compiler = self.get_default_compiler().strip() + else: + self.compiler = compiler.strip() + + self.machpath = os.path.join(self.cimepath, "..", "cime_config", + "machines") + self.macrospath = os.path.join(self.machpath, "cmake_macros", + "Macros.cmake") + + self.machname = self.get_machine_name() + self.machos = self.get_value("OS") + self.mpilibs = self.get_value("MPILIBS").split(",") + + #modifed based on generic_xml.py in CIME + def get_processed_value(self, raw_value, outvar): + + reference_re = re.compile(r"\${?(\w+)}?") + env_ref_re = re.compile(r"\$ENV\{(\w+)\}") + shell_ref_re = re.compile(r"\$SHELL\{([^}]+)\}") + math_re = re.compile(r"\s[+-/*]\s") + item_data = raw_value + + if item_data is None: + return None + + if not isinstance(item_data, str): + return item_data + + for m in env_ref_re.finditer(item_data): + env_var = m.groups()[0] + + if env_var in outvar: + item_data = item_data.replace(m.group(), outvar[env_var]) + + elif env_var in os.environ: + item_data = item_data.replace(m.group(), os.environ[env_var]) + + else: + raise Exception("Undefined env var '{}'".format(env_var)) + + for s in shell_ref_re.finditer(item_data): + shell_cmd = s.groups()[0] + item_data = item_data.replace(s.group(), run_cmd_no_fail(shell_cmd)) + + for m in reference_re.finditer(item_data): + var = m.groups()[0] + ref = self.get_value(var) + + if ref is not None: + item_data = item_data.replace( + m.group(), self.get_processed_value(str(ref), outvar) + ) + elif var == "CIMEROOT": + item_data = item_data.replace(m.group(), self.cimepath) + elif var == "SRCROOT": + item_data = item_data.replace(m.group(), os.path.join(self.cimepath, "..")) + elif var == "USER": + item_data = item_data.replace(m.group(), getpass.getuser()) + + if math_re.search(item_data): + try: + tmp = eval(item_data) + except Exception: + tmp = item_data + item_data = str(tmp) + + return item_data + + #get module info + def get_modules(self, outvar): + + modcmds = [] + outvar["__OMEGA_MODULE_COMMANDS__"] = modcmds + + module_system_node = self.get_child("module_system") + module_system_type = self.get(module_system_node, "type") + + if module_system_type != "module": + print((f"ERROR: '{module_system_type}' " + "module system is not supported.")) + exit(-1) + + out1 = subprocess.check_output("env", shell=True) + env1 = str(out1, 'UTF-8') + + module_nodes = self.get_children( + "modules", root=module_system_node + ) + + modcmd = "module" + + shcmds = [] + + for module_node in module_nodes: + compiler = self.get(module_node, "compiler") + mpilib = self.get(module_node, "mpilib") + debug = self.get(module_node, "DEBUG") + + if not (compiler is None or re.match("^"+compiler+"$", self.compiler)): + continue + + if not (mpilib is None or re.match("^"+mpilib+"$", self.mpilib)): + continue + + if not (debug is None or re.match("^"+debug+"$", self.debug)): + continue + + command_nodes = self.get_children("command", + root=module_node) + for command_node in command_nodes: + name = self.get(command_node, "name") + module = self.text(command_node) + if module is None: + shcmd = f"{modcmd} {name}" + else: + shcmd = f"{modcmd} {name} {module}" + + modcmds.append(shcmd) + shcmds.append(shcmd) + + shcmds.append("env") + + out2 = subprocess.check_output(";".join(shcmds), shell=True) + env2 = str(out2, 'UTF-8') + + parsed1 = {} + + for (name, value) in pat_envvar.findall(env1): + parsed1[name] = value + + for (name, value) in pat_envvar.findall(env2): + if name in parsed1: + if parsed1[name] != value: + outvar[name] = value + else: + outvar[name] = value + + #get environmental variables info + def get_envs(self, outvar): + + exports = {} + outvar["__OMEGA_SCRIPT_EXPORTS__"] = exports + + envvar_nodes = self.get_children("environment_variables", root=self.machine_node) + + #possible attribs : compiler, DEBUG, SMP_PRESENT, mpilib + for envvar_node in envvar_nodes: + compiler = self.get(envvar_node, "compiler") + mpilib = self.get(envvar_node, "mpilib") + debug = self.get(envvar_node, "DEBUG") + + if not (compiler is None or re.match("^"+compiler+"$", self.compiler)): + continue + + if not (mpilib is None or re.match("^"+mpilib+"$", self.mpilib)): + continue + + if not (debug is None or re.match("^"+debug+"$", self.debug)): + continue + + env_nodes = self.get_children("env", root=envvar_node) + for env_node in env_nodes: + name = self.get(env_node, "name") + value = self.get_processed_value(self.text(env_node).strip(), outvar) + if value is None: + outvar[name] = "" + exports[name] = outvar[name] + elif value.startswith("$ENV{") and value.endswith("}"): + vname = value[5:-1] + if vname in outvar: + outvar[name] = outvar[vname] + elif vname in os.environ: + outvar[name] = os.environ[vname] + else: + outvar[name] = "" + exports[name] = outvar[name] + elif value.startswith("$SHELL{") and value.endswith("}"): + print("Warning: SHELL evaluation is not supported.") + + else: + outvar[name] = value + exports[name] = outvar[name] + + #get mpirun info + def get_mpirun(self, outvar): + + mpirun_node = self.get_child("mpirun") + mpirun_mpilib = self.get(mpirun_node, "mpilib") + + exec_nodes = self.get_children( + "executable", root=mpirun_node + ) + + if len(exec_nodes) != 1: + print("ERROR: 'more than one executable nodes in mpirun node") + exit(-1) + + self.mpilib = mpirun_mpilib + if self.mpilib == "default": + self.mpilib = self.mpilibs[0] + + self.mpiexec = self.text(exec_nodes[0]) + + #collect machine info + def gen_machinfo(self): + + outvar = {} + + self.get_mpirun(outvar) + self.get_modules(outvar) + self.get_envs(outvar) + self.write_output(outvar) + self.generate_scripts(outvar) + + #create a temporary cmake script to be included + #in the main Omega cmake build system + def write_output(self, outvar): + + with open(self.outpath, "w") as f: + f.write("message(STATUS \"Reading E3SM machine info\")\n") + + for key, value in outvar.items(): + if not key.startswith("__OMEGA_"): + f.write("set(ENV{%s} \"%s\")\n" % (key, value)) + + f.write(f"set(MACH {self.machname})\n") + f.write(f"set(OS {self.machos})\n") + f.write(f"set(COMPILER {self.compiler})\n") + f.write(f"set(MPI_EXEC {self.mpiexec})\n") + f.write(f"set(CASEROOT {self.machpath})\n") + f.write(f"include({self.macrospath})\n") + + #create scripts + def generate_scripts(self, outvar): + + omega_env = os.path.join(os.path.dirname(self.outpath), "omega_env.sh") + omega_build = os.path.join(os.path.dirname(self.outpath), "omega_build.sh") + omega_run = os.path.join(os.path.dirname(self.outpath), "omega_run.sh") + omega_ctest = os.path.join(os.path.dirname(self.outpath), "omega_ctest.sh") + + with open(omega_env, "w") as f: + f.write("#!/usr/bin/env bash\n\n") + + f.write("# module commands\n") + for cmd in outvar["__OMEGA_MODULE_COMMANDS__"]: + f.write(cmd + "\n") + + f.write("\n# env. variables\n") + for key, value in outvar["__OMEGA_SCRIPT_EXPORTS__"].items(): + f.write("export %s=\"%s\"\n" % (key, value)) + + with open(omega_build, "w") as f: + f.write("#!/usr/bin/env bash\n\n") + + f.write("source ./omega_env.sh\n") + f.write("make\n") + + with open(omega_run, "w") as f: + f.write("#!/usr/bin/env bash\n\n") + + f.write("source ./omega_env.sh\n") + f.write("./src/omega.exe\n") + + with open(omega_ctest, "w") as f: + f.write("#!/usr/bin/env bash\n\n") + + f.write("source ./omega_env.sh\n") + f.write("ctest # --rerun-failed --output-on-failure\n") + + st = os.stat(omega_env) + os.chmod(omega_env, st.st_mode | stat.S_IEXEC) + + st = os.stat(omega_build) + os.chmod(omega_build, st.st_mode | stat.S_IEXEC) + + st = os.stat(omega_run) + os.chmod(omega_run, st.st_mode | stat.S_IEXEC) + + st = os.stat(omega_ctest) + os.chmod(omega_ctest, st.st_mode | stat.S_IEXEC) + + +def main(): + + mach = OmegaMachines(args.cimepath, args.outpath, args.machine, + args.compiler, args.debug) + mach.gen_machinfo() + + +if __name__ == "__main__": + main() diff --git a/components/omega/doc/devGuide/CMakeBuild.md b/components/omega/doc/devGuide/CMakeBuild.md index 490aab49520a..b45e0958bbbe 100644 --- a/components/omega/doc/devGuide/CMakeBuild.md +++ b/components/omega/doc/devGuide/CMakeBuild.md @@ -15,6 +15,7 @@ The build step consists of adding three subdirectories that drive builds for external libraries, the Omega model, and optional tests. Python is required to use this build system. +The version of CMake should be 3.21 or later for supporting HIP. ## Step 1: Setup @@ -49,10 +50,9 @@ OMEGA_ARCH: User-defined programming framework (e.g., "CUDA", "HIP", "OPENMP", " OMEGA_${YAKL_ARCH}_FLAGS: Framework-specific compiler flags OMEGA_CXX_COMPILER: C++ compiler OMEGA_CIME_COMPILER: E3SM compiler name defined in config_machines.xml +OMEGA_CIME_MACHINE: E3SM machine name defined in config_machines.xml OMEGA_CXX_FLAGS: a list for C++ compiler flags OMEGA_LINK_OPTIONS: a list for linker flags -OMEGA_INCLUDE_DIRECTORIES: Directory where header files are located -OMEGA_LINK_DIRECTORIES: Directory where library files are located OMEGA_BUILD_EXECUTABLE: Enable building the Omega executable OMEGA_BUILD_TEST: Enable building Omega tests ``` @@ -71,13 +71,14 @@ CMake variables ``` CMAKE_CURRENT_SOURCE_DIR +CMAKE_CURRENT_BINARY_DIR +CMAKE_CURRENT_LIST_DIR CMAKE_CXX_STANDARD CMAKE_CXX_COMPILER CMAKE_CXX_FLAGS CMAKE_CURRENT_LIST_DIR CMAKE_BUILD_TYPE CMAKE_INSTALL_PREFIX -CMAKE_CURRENT_BINARY_DIR CMAKE_VERSION ``` diff --git a/components/omega/doc/userGuide/OmegaBuild.md b/components/omega/doc/userGuide/OmegaBuild.md index 4c86b3038757..e56d5a0c5bd7 100644 --- a/components/omega/doc/userGuide/OmegaBuild.md +++ b/components/omega/doc/userGuide/OmegaBuild.md @@ -14,6 +14,7 @@ based on the computing system where the build is taking place, as well as user input from the CMake command-line. For the Omega build system to function, a Python interpreter is necessary. +The minimum version of CMake is 3.21. ## Standalone Build @@ -33,6 +34,16 @@ as illustrated below: ${E3SM_HOME}/components/omega ``` +Once the command completes succwssfuly, several scripts will be created in +the build directory: + +* omega\_env.sh : load specific modules and set env. variables read from CIME +* omega\_build.sh : run `make` command after sourcing `omega\_env.sh` +* omega\_run.sh : run `./src/omega.exe` after sourcing `omega\_env.sh` +* omega\_ctest.sh : run `ctest` after sourcing `omega\_env.sh` + +Run omega\_build.sh in the build directory to build Omega. + To employ a specific compiler, users can utilize the `OMEGA_CXX_COMPILER` CMake variable, providing CMake with the compiler's path. @@ -44,6 +55,21 @@ CMake variable, providing CMake with the compiler's path. `OMEGA_CXX_COMPILER` overrides `OMEGA_CIME_COMPILER`. +To build Omega for GPU, add OMEGA\_ARCH to one of "CUDA" or "HIP". + +```sh +>> cmake \ + -DOMEGA_CIME_COMPILER=nvidiagpu \ + -DOMEGA_CIME_MACHINE=pm-gpu \ + -DOMEGA_ARCH=CUDA \ + ${E3SM_HOME}/components/omega +``` +In some cases, you may want to add OMEGA\_CIME\_MACHINE to specify which +system you intend to use. + +The values of OMEGA\_CIME\_COMPILER and OMEGA\_CIME\_MACHINE are defined in +"${E3SM}/cime\_config/machines/config\_machines.xml". + To enable the ctest-based unittest option, include the `OMEGA_BUILD_TEST` option as illustrated below. diff --git a/components/omega/external/CMakeLists.txt b/components/omega/external/CMakeLists.txt index 3b1b3e05a403..07880188c9fe 100644 --- a/components/omega/external/CMakeLists.txt +++ b/components/omega/external/CMakeLists.txt @@ -1,7 +1,17 @@ # Add external packages +# Add the spdlog library +add_subdirectory( + ${E3SM_EXTERNALS_ROOT}/ekat/extern/spdlog + ${CMAKE_CURRENT_BINARY_DIR}/ekat/extern/spdlog +) + # Add the YAKL library -add_subdirectory(${E3SM_EXTERNALS_ROOT}/YAKL ${CMAKE_CURRENT_BINARY_DIR}/YAKL) +add_subdirectory( + ${E3SM_EXTERNALS_ROOT}/YAKL + ${CMAKE_CURRENT_BINARY_DIR}/YAKL +) + if (CMAKE_VERSION VERSION_GREATER "3.18.0") set_property(TARGET yakl PROPERTY CUDA_ARCHITECTURES OFF) endif() @@ -9,6 +19,3 @@ endif() if(OMEGA_DEBUG) target_compile_definitions(yakl INTERFACE YAKL_DEBUG=1) endif() - -# Add the spdlog library -add_subdirectory(${E3SM_EXTERNALS_ROOT}/ekat/extern/spdlog ${CMAKE_CURRENT_BINARY_DIR}/ekat/extern/spdlog) diff --git a/components/omega/gen_machine_info.py b/components/omega/gen_machine_info.py deleted file mode 100644 index 9892384c0506..000000000000 --- a/components/omega/gen_machine_info.py +++ /dev/null @@ -1,194 +0,0 @@ -import argparse -import os -import re -import subprocess -import sys -import typing - -pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) - - -def parse_cmdline(): - - here, progname = os.path.split(__file__) - - parser = argparse.ArgumentParser( - prog=progname, - description='generate machine specific infomation', - epilog='Contact: ') - - parser.add_argument('-p', '--cimepath', default=os.path.realpath( - os.path.join(here, "..", "..", "cime"))) # CIME root - parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile - parser.add_argument('-c', '--compiler') # compiler - parser.add_argument('-v', '--verbose') # verbose output - - # cimepath, outpath, compiler, verbose - return parser.parse_args() - - -args = parse_cmdline() - -sys.path.insert(0, args.cimepath) -from CIME.XML.machines import Machines # noqa: E402 - -sys.path.pop(0) - - -class OmegaMachines(Machines): - - def __init__(self, cimepath, outpath, compiler): - - super(OmegaMachines, self).__init__() - - self.cimepath = cimepath - self.outpath = outpath - self.mpiexec = "" - - if compiler is None: - self.compiler = self.get_default_compiler().strip() - else: - self.compiler = compiler.strip() - - self.machpath = os.path.join(self.cimepath, "..", "cime_config", - "machines") - self.macrospath = os.path.join(self.machpath, "cmake_macros", - "Macros.cmake") - - self.machname = self.get_machine_name() - self.machos = self.get_value("OS") - - def get_modules(self, outvar): - - module_system_node = self.get_child("module_system") - module_system_type = self.get(module_system_node, "type") - - # check = (True if self.get(module_system_node, - # "allow_error") == "true" else False) - - if module_system_type != "module": - print((f"ERROR: '{module_system_type}' " - "module system is not supported.")) - exit(-1) - - out1 = subprocess.check_output("env", shell=True) - env1 = str(out1, 'UTF-8') - - module_nodes = self.get_children( - "modules", root=module_system_node - ) - - modcmd = "module" - - shcmds = [] - - for module_node in module_nodes: - compiler = self.get(module_node, "compiler") - if compiler is None or re.match(compiler, self.compiler): - command_nodes = self.get_children("command", - root=module_node) - for command_node in command_nodes: - name = self.get(command_node, "name") - module = self.text(command_node) - if module is None: - shcmds.append(f"{modcmd} {name}") - else: - shcmds.append(f"{modcmd} {name} {module}") - - shcmds.append("env") - - out2 = subprocess.check_output(";".join(shcmds), shell=True) - env2 = str(out2, 'UTF-8') - - parsed1 = {} - - for (name, value) in pat_envvar.findall(env1): - parsed1[name] = value - - for (name, value) in pat_envvar.findall(env2): - if name in parsed1: - if parsed1[name] != value: - outvar[name] = value - else: - outvar[name] = value - - def get_envs(self, outvar): - - envvar_nodes = self.get_children( - root=self.get_child("environment_variables")) - - for envvar_node in envvar_nodes: - compiler = self.get(envvar_node, "compiler") - if compiler is None or re.match(compiler, self.compiler): - env_nodes = self.get_children("env", root=envvar_node) - for env_node in env_nodes: - name = self.get(env_node, "name") - value = self.text(env_node).strip() - if value is None: - outvar[name] = "" - elif value.startswith("$ENV{") and value.endswith("}"): - vname = value[5:-1] - if vname in outvar: - outvar[name] = outvar[vname] - elif vname in os.environ: - outvar[name] = os.environ[vname] - else: - outvar[name] = "" - else: - outvar[name] = value - - def get_mpirun(self, outvar): - - mpirun_node = self.get_child("mpirun") - mpirun_mpilib = self.get(mpirun_node, "mpilib") - - if mpirun_mpilib != "default": - print(f"ERROR: '{mpirun_mpilib}' mpilib is not supported.") - exit(-1) - - exec_nodes = self.get_children( - "executable", root=mpirun_node - ) - - if len(exec_nodes) != 1: - print("ERROR: 'more than one executable nodes in mpirun node") - exit(-1) - - self.mpiexec = self.text(exec_nodes[0]) - - def gen_machinfo(self): - - outvar: typing.Dict[str, int] = {} - - self.get_modules(outvar) - self.get_envs(outvar) - self.get_mpirun(outvar) - self.write_output(outvar) - - def write_output(self, outvar): - - with open(self.outpath, "w") as f: - f.write("message(STATUS \"Reading E3SM machine info\")\n") - - for key, value in outvar.items(): - f.write("set(ENV{%s} \"%s\")\n" % (key, value)) - - f.write(f"set(MACH {self.machname})\n") - f.write(f"set(OS {self.machos})\n") - f.write(f"set(COMPILER {self.compiler})\n") - f.write(f"set(MPI_EXEC {self.mpiexec})\n") - f.write(f"set(CASEROOT {self.machpath})\n") - f.write(f"include({self.macrospath})\n") - - f.write(("message(STATUS \"End of reading E3SM " - "machine info\")\n")) - - -def main(): - - mach = OmegaMachines(args.cimepath, args.outpath, args.compiler) - mach.gen_machinfo() - - -if __name__ == "__main__": - main() diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt index d50347312ba6..9cfde656a0a1 100644 --- a/components/omega/src/CMakeLists.txt +++ b/components/omega/src/CMakeLists.txt @@ -5,21 +5,14 @@ file(GLOB _LIBSRC_FILES base/*.cpp infra/*.cpp ocn/*.cpp) # Create the library target add_library(${OMEGA_LIB_NAME} ${_LIBSRC_FILES}) -target_link_libraries(${OMEGA_LIB_NAME} spdlog "${OMEGA_LINK_FLAGS}") # add include directories -target_include_directories( +target_compile_options( ${OMEGA_LIB_NAME} PRIVATE - "${OMEGA_SOURCE_DIR}/src/base" - "${OMEGA_SOURCE_DIR}/src/infra" - ${OMEGA_INCLUDE_DIRECTORIES} -) - -target_link_directories( - ${OMEGA_LIB_NAME} - PRIVATE - ${OMEGA_LINK_DIRECTORIES} + "-I${OMEGA_SOURCE_DIR}/src/base" + "-I${OMEGA_SOURCE_DIR}/src/infra" + ${OMEGA_CXX_FLAGS} ) target_link_options( @@ -28,44 +21,25 @@ target_link_options( ${OMEGA_LINK_OPTIONS} ) -# include yakl cmake utility -include(${E3SM_EXTERNALS_ROOT}/YAKL/yakl_utils.cmake) - -# help the library target to be built with yakl -yakl_process_target(${OMEGA_LIB_NAME}) - -# handles cuda cases -if (YAKL_ARCH STREQUAL "CUDA") - set_target_properties(${OMEGA_LIB_NAME} PROPERTIES LINKER_LANGUAGE CXX) - if (CMAKE_VERSION VERSION_GREATER "3.18.0") - set_target_properties(${OMEGA_LIB_NAME} PROPERTIES CUDA_ARCHITECTURES OFF) - endif() -endif() +target_link_libraries(${OMEGA_LIB_NAME} spdlog yakl) # build Omega executable if(OMEGA_BUILD_EXECUTABLE) + set(EXESRC_FILES drivers/DrvDummy.cpp ) + # Create the executable target add_executable(${OMEGA_EXE_NAME} ${EXESRC_FILES}) - target_include_directories( + + target_compile_options( ${OMEGA_EXE_NAME} PRIVATE - "${OMEGA_SOURCE_DIR}/src/base" - "${CMAKE_CURRENT_SOURCE_DIR}/infra" + "-L${OMEGA_SOURCE_DIR}/src/base" + "-L${CMAKE_CURRENT_SOURCE_DIR}/infra" ) - target_link_libraries(${OMEGA_EXE_NAME} ${OMEGA_LIB_NAME}) - - # help the executable target to be built with yakl - yakl_process_target(${OMEGA_EXE_NAME}) - # handles cuda cases - if (YAKL_ARCH STREQUAL "CUDA") - set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX) - if (CMAKE_VERSION VERSION_GREATER "3.18.0") - set_target_properties(${OMEGA_EXE_NAME} PROPERTIES CUDA_ARCHITECTURES OFF) - endif() - endif() + target_link_libraries(${OMEGA_EXE_NAME} ${OMEGA_LIB_NAME}) endif() diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index c5bcca4b78ac..114ada6cf8db 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -1,59 +1,122 @@ # Omega Unit Tests -# Add data type test +################## +# Data type test +################## set(_TestDataTypesName testDataTypes.exe) add_executable(${_TestDataTypesName} base/DataTypesTest.cpp) -target_link_libraries(${_TestDataTypesName} ${OMEGA_LIB_NAME}) -target_include_directories( - ${_TestDataTypesName} - PRIVATE - "${OMEGA_SOURCE_DIR}/src/base" - ${OMEGA_INCLUDE_DIRECTORIES} + +target_compile_options( + ${_TestDataTypesName} + PRIVATE + "-I${OMEGA_SOURCE_DIR}/src/base" + "-I${OMEGA_SOURCE_DIR}/src/infra" + ${OMEGA_CXX_FLAGS} +) + +target_link_options( + ${_TestDataTypesName} + PRIVATE + ${OMEGA_LINK_OPTIONS} +) + +target_link_libraries(${_TestDataTypesName} ${OMEGA_LIB_NAME} yakl) + +add_test( + NAME DATA_TYPES_TEST + COMMAND ${MPI_EXEC} -n 1 -- ./${_TestDataTypesName} ) -target_link_directories( - ${_TestDataTypesName} - PRIVATE - ${OMEGA_LINK_DIRECTORIES} + +################## +# Machine env test +################## + +set(_TestMachEnvName testMachEnv.exe) + +add_executable(${_TestMachEnvName} base/MachEnvTest.cpp) + +target_compile_options( + ${_TestMachEnvName} + PRIVATE + "-I${OMEGA_SOURCE_DIR}/src/base" + "-I${OMEGA_SOURCE_DIR}/src/infra" + ${OMEGA_CXX_FLAGS} ) target_link_options( - ${_TestDataTypesName} - PRIVATE - ${OMEGA_LINK_OPTIONS} + ${_TestMachEnvName} + PRIVATE + ${OMEGA_LINK_OPTIONS} +) + +target_link_libraries(${_TestMachEnvName} ${OMEGA_LIB_NAME} yakl) + +add_test( + NAME MACHINE_ENV_TEST + COMMAND ${MPI_EXEC} -n 8 -- ./${_TestMachEnvName} ) -add_test(NAME DATA_TYPES_TEST COMMAND ${_TestDataTypesName}) -# Add machine env test -# NOTE: disabled temporary due to not-launching mininum 8 ranks -#add_executable(testMachEnv.exe base/MachEnvTest.cpp) -#target_link_libraries(testMachEnv.exe ${OMEGA_LIB_NAME}) -#target_include_directories( -# testMachEnv.exe -# PRIVATE -# "${OMEGA_SOURCE_DIR}/src/base" -#) -#add_test(NAME MACHINE_ENV_TEST COMMAND testMachEnv.exe) +################## +# Logging test +################## set(_TestLoggingName testLogging.exe) add_executable(${_TestLoggingName} infra/LoggingTest.cpp) -target_link_libraries(${_TestLoggingName} ${OMEGA_LIB_NAME}) -target_include_directories( - ${_TestLoggingName} - PRIVATE - "${OMEGA_SOURCE_DIR}/src/base" - "${OMEGA_SOURCE_DIR}/src/infra" + +target_compile_options( + ${_TestLoggingName} + PRIVATE + "-I${OMEGA_SOURCE_DIR}/src/base" + "-I${OMEGA_SOURCE_DIR}/src/infra" + ${OMEGA_CXX_FLAGS} +) + +target_link_options( + ${_TestLoggingName} + PRIVATE + ${OMEGA_LINK_OPTIONS} +) + +target_link_libraries(${_TestLoggingName} ${OMEGA_LIB_NAME} yakl) + +add_test(NAME LOGGING_TEST COMMAND ./${_TestLoggingName}) + +################## +# YAKL test +################## + +set(_TestYaklName testYakl.exe) + +add_executable( + ${_TestYaklName} ${E3SM_EXTERNALS_ROOT}/YAKL/unit/performance/performance.cpp ) -add_test(NAME LOGGING_TEST COMMAND testLogging.exe) +include(${E3SM_EXTERNALS_ROOT}/YAKL/yakl_utils.cmake) +yakl_process_target(${_TestYaklName}) + +# handles cuda cases +if (YAKL_ARCH STREQUAL "CUDA") + set_target_properties(${_TestYaklName} PROPERTIES LINKER_LANGUAGE CXX) + if (CMAKE_VERSION VERSION_GREATER "3.18.0") + set_target_properties(${_TestYaklName} PROPERTIES CUDA_ARCHITECTURES OFF) + endif() +endif() + +add_test(NAME YAKL_TEST COMMAND ./${_TestYaklName}) + +################## +# test properties +################## -# set test properties set_tests_properties( - DATA_TYPES_TEST -# MACHINE_ENV_TEST - LOGGING_TEST - PROPERTIES FAIL_REGULAR_EXPRESSION "FAIL") + DATA_TYPES_TEST + MACHINE_ENV_TEST + LOGGING_TEST + YAKL_TEST + PROPERTIES FAIL_REGULAR_EXPRESSION "FAIL" +) From 9e3120b69c8914c5c3c1869fcd498c0e8f59b72b Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sat, 7 Oct 2023 16:01:54 -0400 Subject: [PATCH 05/10] fixed format issues --- components/omega/create_scripts.py | 92 ++++++++++++-------- components/omega/doc/userGuide/OmegaBuild.md | 4 +- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py index 1efe1838ebd8..cfadd0faddb6 100644 --- a/components/omega/create_scripts.py +++ b/components/omega/create_scripts.py @@ -6,10 +6,12 @@ import argparse import os import re +import stat import subprocess import sys import typing -import stat +import getpass + pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) @@ -24,32 +26,33 @@ def parse_cmdline(): epilog='Contact: ') parser.add_argument('-p', '--cimepath', default=os.path.realpath( - os.path.join(here, "..", "..", "cime"))) # CIME root - parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile - parser.add_argument('-m', '--machine') # machine - parser.add_argument('-c', '--compiler') # compiler + os.path.join(here, "..", "..", "cime"))) # CIME root + parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile + parser.add_argument('-m', '--machine') # machine + parser.add_argument('-c', '--compiler') # compiler parser.add_argument('-d', '--debug', action='store_const', - const="TRUE", default="FALSE") # debug mode - parser.add_argument('-v', '--verbose') # verbose output + const="TRUE", default="FALSE") # debug mode + parser.add_argument('-v', '--verbose') # verbose output return parser.parse_args() + args = parse_cmdline() -#import Machines class from CIME +# import Machines class from CIME sys.path.insert(0, args.cimepath) from CIME.XML.machines import Machines # noqa: E402 sys.path.pop(0) -#run a shell command +# run a shell command def run_cmd_no_fail(cmd): out = subprocess.run(cmd, shell=True, capture_output=True) return str(out.stdout, 'UTF-8') -#main class that extends Machines class +# main class that extends Machines class class OmegaMachines(Machines): def __init__(self, cimepath, outpath, machine, compiler, debug): @@ -104,24 +107,26 @@ def get_processed_value(self, raw_value, outvar): item_data = item_data.replace(m.group(), os.environ[env_var]) else: - raise Exception("Undefined env var '{}'".format(env_var)) + raise Exception(f"Undefined env var '{env_var}'") for s in shell_ref_re.finditer(item_data): shell_cmd = s.groups()[0] - item_data = item_data.replace(s.group(), run_cmd_no_fail(shell_cmd)) + item_data = item_data.replace( + s.group(), run_cmd_no_fail(shell_cmd) + ) for m in reference_re.finditer(item_data): var = m.groups()[0] ref = self.get_value(var) if ref is not None: - item_data = item_data.replace( - m.group(), self.get_processed_value(str(ref), outvar) - ) + procval = self.get_processed_value(str(ref), outvar) + item_data = item_data.replace(m.group(), procval) elif var == "CIMEROOT": item_data = item_data.replace(m.group(), self.cimepath) elif var == "SRCROOT": - item_data = item_data.replace(m.group(), os.path.join(self.cimepath, "..")) + cpath = os.path.join(self.cimepath, "..") + item_data = item_data.replace(m.group(), cpath) elif var == "USER": item_data = item_data.replace(m.group(), getpass.getuser()) @@ -134,12 +139,12 @@ def get_processed_value(self, raw_value, outvar): return item_data - #get module info + # get module info def get_modules(self, outvar): modcmds = [] outvar["__OMEGA_MODULE_COMMANDS__"] = modcmds - + module_system_node = self.get_child("module_system") module_system_type = self.get(module_system_node, "type") @@ -164,13 +169,16 @@ def get_modules(self, outvar): mpilib = self.get(module_node, "mpilib") debug = self.get(module_node, "DEBUG") - if not (compiler is None or re.match("^"+compiler+"$", self.compiler)): + if not (compiler is None or + re.match("^" + compiler + "$", self.compiler)): continue - if not (mpilib is None or re.match("^"+mpilib+"$", self.mpilib)): + if not (mpilib is None or + re.match("^" + mpilib + "$", self.mpilib)): continue - if not (debug is None or re.match("^"+debug+"$", self.debug)): + if not (debug is None or + re.match("^" + debug + "$", self.debug)): continue command_nodes = self.get_children("command", @@ -203,27 +211,31 @@ def get_modules(self, outvar): else: outvar[name] = value - #get environmental variables info + # get environmental variables info def get_envs(self, outvar): exports = {} outvar["__OMEGA_SCRIPT_EXPORTS__"] = exports - envvar_nodes = self.get_children("environment_variables", root=self.machine_node) + envvar_nodes = self.get_children("environment_variables", + root=self.machine_node) - #possible attribs : compiler, DEBUG, SMP_PRESENT, mpilib + # possible attribs : compiler, DEBUG, SMP_PRESENT, mpilib for envvar_node in envvar_nodes: compiler = self.get(envvar_node, "compiler") mpilib = self.get(envvar_node, "mpilib") debug = self.get(envvar_node, "DEBUG") - if not (compiler is None or re.match("^"+compiler+"$", self.compiler)): + if not (compiler is None or + re.match("^" + compiler + "$", self.compiler)): continue - if not (mpilib is None or re.match("^"+mpilib+"$", self.mpilib)): + if not (mpilib is None or + re.match("^" + mpilib + "$", self.mpilib)): continue - if not (debug is None or re.match("^"+debug+"$", self.debug)): + if not (debug is None or + re.match("^" + debug + "$", self.debug)): continue env_nodes = self.get_children("env", root=envvar_node) @@ -249,7 +261,7 @@ def get_envs(self, outvar): outvar[name] = value exports[name] = outvar[name] - #get mpirun info + # get mpirun info def get_mpirun(self, outvar): mpirun_node = self.get_child("mpirun") @@ -269,7 +281,7 @@ def get_mpirun(self, outvar): self.mpiexec = self.text(exec_nodes[0]) - #collect machine info + # collect machine info def gen_machinfo(self): outvar = {} @@ -280,8 +292,8 @@ def gen_machinfo(self): self.write_output(outvar) self.generate_scripts(outvar) - #create a temporary cmake script to be included - #in the main Omega cmake build system + # create a temporary cmake script to be included + # in the main Omega cmake build system def write_output(self, outvar): with open(self.outpath, "w") as f: @@ -298,13 +310,17 @@ def write_output(self, outvar): f.write(f"set(CASEROOT {self.machpath})\n") f.write(f"include({self.macrospath})\n") - #create scripts + # create scripts def generate_scripts(self, outvar): - omega_env = os.path.join(os.path.dirname(self.outpath), "omega_env.sh") - omega_build = os.path.join(os.path.dirname(self.outpath), "omega_build.sh") - omega_run = os.path.join(os.path.dirname(self.outpath), "omega_run.sh") - omega_ctest = os.path.join(os.path.dirname(self.outpath), "omega_ctest.sh") + omega_env = os.path.join(os.path.dirname(self.outpath), + "omega_env.sh") + omega_build = os.path.join(os.path.dirname(self.outpath), + "omega_build.sh") + omega_run = os.path.join(os.path.dirname(self.outpath), + "omega_run.sh") + omega_ctest = os.path.join(os.path.dirname(self.outpath), + "omega_ctest.sh") with open(omega_env, "w") as f: f.write("#!/usr/bin/env bash\n\n") @@ -315,7 +331,7 @@ def generate_scripts(self, outvar): f.write("\n# env. variables\n") for key, value in outvar["__OMEGA_SCRIPT_EXPORTS__"].items(): - f.write("export %s=\"%s\"\n" % (key, value)) + f.write(f"export {key}=\"{value}\"\n") with open(omega_build, "w") as f: f.write("#!/usr/bin/env bash\n\n") @@ -351,7 +367,7 @@ def generate_scripts(self, outvar): def main(): mach = OmegaMachines(args.cimepath, args.outpath, args.machine, - args.compiler, args.debug) + args.compiler, args.debug) mach.gen_machinfo() diff --git a/components/omega/doc/userGuide/OmegaBuild.md b/components/omega/doc/userGuide/OmegaBuild.md index e56d5a0c5bd7..b0641af345ae 100644 --- a/components/omega/doc/userGuide/OmegaBuild.md +++ b/components/omega/doc/userGuide/OmegaBuild.md @@ -34,8 +34,8 @@ as illustrated below: ${E3SM_HOME}/components/omega ``` -Once the command completes succwssfuly, several scripts will be created in -the build directory: +Once the command completes successfully, several scripts will be created +in the build directory. * omega\_env.sh : load specific modules and set env. variables read from CIME * omega\_build.sh : run `make` command after sourcing `omega\_env.sh` From 1546f844b4854e645cf0884131e13984b6ca243a Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sat, 7 Oct 2023 16:12:09 -0400 Subject: [PATCH 06/10] fixing format issues in create_scripts.py --- components/omega/create_scripts.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py index cfadd0faddb6..853888fa36ab 100644 --- a/components/omega/create_scripts.py +++ b/components/omega/create_scripts.py @@ -4,13 +4,12 @@ """ import argparse +import getpass import os import re import stat import subprocess import sys -import typing -import getpass pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) @@ -26,13 +25,13 @@ def parse_cmdline(): epilog='Contact: ') parser.add_argument('-p', '--cimepath', default=os.path.realpath( - os.path.join(here, "..", "..", "cime"))) # CIME root - parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile - parser.add_argument('-m', '--machine') # machine - parser.add_argument('-c', '--compiler') # compiler + os.path.join(here, "..", "..", "cime"))) # CIME root + parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile + parser.add_argument('-m', '--machine') # machine + parser.add_argument('-c', '--compiler') # compiler parser.add_argument('-d', '--debug', action='store_const', - const="TRUE", default="FALSE") # debug mode - parser.add_argument('-v', '--verbose') # verbose output + const="TRUE", default="FALSE") # debug mode + parser.add_argument('-v', '--verbose') # verbose output return parser.parse_args() @@ -82,7 +81,7 @@ def __init__(self, cimepath, outpath, machine, compiler, debug): self.machos = self.get_value("OS") self.mpilibs = self.get_value("MPILIBS").split(",") - #modifed based on generic_xml.py in CIME + # modifed based on generic_xml.py in CIME def get_processed_value(self, raw_value, outvar): reference_re = re.compile(r"\${?(\w+)}?") @@ -111,9 +110,8 @@ def get_processed_value(self, raw_value, outvar): for s in shell_ref_re.finditer(item_data): shell_cmd = s.groups()[0] - item_data = item_data.replace( - s.group(), run_cmd_no_fail(shell_cmd) - ) + item_data = item_data.replace(s.group(), + run_cmd_no_fail(shell_cmd)) for m in reference_re.finditer(item_data): var = m.groups()[0] @@ -237,11 +235,12 @@ def get_envs(self, outvar): if not (debug is None or re.match("^" + debug + "$", self.debug)): continue - + env_nodes = self.get_children("env", root=envvar_node) for env_node in env_nodes: name = self.get(env_node, "name") - value = self.get_processed_value(self.text(env_node).strip(), outvar) + value = self.get_processed_value(self.text(env_node).strip(), + outvar) if value is None: outvar[name] = "" exports[name] = outvar[name] From ab592e851b55b53baca6e946f442e797f4c69a9e Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sat, 7 Oct 2023 16:23:31 -0400 Subject: [PATCH 07/10] annotated type for python containers --- components/omega/create_scripts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py index 853888fa36ab..484d5c4624e9 100644 --- a/components/omega/create_scripts.py +++ b/components/omega/create_scripts.py @@ -9,6 +9,7 @@ import re import stat import subprocess +import typing import sys @@ -140,7 +141,7 @@ def get_processed_value(self, raw_value, outvar): # get module info def get_modules(self, outvar): - modcmds = [] + modcmds: typing.List[str] = [] outvar["__OMEGA_MODULE_COMMANDS__"] = modcmds module_system_node = self.get_child("module_system") @@ -212,7 +213,7 @@ def get_modules(self, outvar): # get environmental variables info def get_envs(self, outvar): - exports = {} + exports: typing.Dict[str, str] = {} outvar["__OMEGA_SCRIPT_EXPORTS__"] = exports envvar_nodes = self.get_children("environment_variables", @@ -283,7 +284,7 @@ def get_mpirun(self, outvar): # collect machine info def gen_machinfo(self): - outvar = {} + outvar: typing.Dict[str, str] = {} self.get_mpirun(outvar) self.get_modules(outvar) From 596eae9eb63b4f836eebbeb950417cdf313034c8 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Mon, 9 Oct 2023 13:02:47 -0400 Subject: [PATCH 08/10] fixed issues of formatting --- components/omega/create_scripts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py index 484d5c4624e9..89df36b4f52f 100644 --- a/components/omega/create_scripts.py +++ b/components/omega/create_scripts.py @@ -9,9 +9,9 @@ import re import stat import subprocess -import typing import sys +import typing pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) @@ -175,7 +175,7 @@ def get_modules(self, outvar): if not (mpilib is None or re.match("^" + mpilib + "$", self.mpilib)): continue - + if not (debug is None or re.match("^" + debug + "$", self.debug)): continue @@ -232,7 +232,7 @@ def get_envs(self, outvar): if not (mpilib is None or re.match("^" + mpilib + "$", self.mpilib)): continue - + if not (debug is None or re.match("^" + debug + "$", self.debug)): continue From 5609d149db46468347b4a1bb98b0d0eeec22e216 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Mon, 9 Oct 2023 13:12:29 -0400 Subject: [PATCH 09/10] fixed isort issue in create_scripts.py --- components/omega/create_scripts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py index 89df36b4f52f..51bbed1821a1 100644 --- a/components/omega/create_scripts.py +++ b/components/omega/create_scripts.py @@ -10,7 +10,6 @@ import stat import subprocess import sys - import typing pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) From d34b3449fc5ee05c00076cd2a303af5f7c3f2564 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Tue, 10 Oct 2023 11:51:12 -0400 Subject: [PATCH 10/10] minor update in OmegaGuild user guide --- components/omega/doc/userGuide/OmegaBuild.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/components/omega/doc/userGuide/OmegaBuild.md b/components/omega/doc/userGuide/OmegaBuild.md index b0641af345ae..e37ce6e29f25 100644 --- a/components/omega/doc/userGuide/OmegaBuild.md +++ b/components/omega/doc/userGuide/OmegaBuild.md @@ -26,11 +26,15 @@ that directory. To utilize a compiler name that's defined in the E3SM machine configuration file (`config_machines.xml`), employ the `OMEGA_CIME_COMPILER` CMake variable, -as illustrated below: +as illustrated below. In some cases, you may also want to add `OMEGA_CIME_MACHINE` +to specify which system you intend to use. The values of `OMEGA_CIME_COMPILER` +and `OMEGA_CIME_MACHINE` are defined in +"${E3SM}/cime\_config/machines/config\_machines.xml". ```sh >> cmake \ - -DOMEGA_CIME_COMPILER=crayclang \ + -DOMEGA_CIME_COMPILER=nvidiagpu \ + -DOMEGA_CIME_MACHINE=pm-gpu \ ${E3SM_HOME}/components/omega ``` @@ -38,9 +42,9 @@ Once the command completes successfully, several scripts will be created in the build directory. * omega\_env.sh : load specific modules and set env. variables read from CIME -* omega\_build.sh : run `make` command after sourcing `omega\_env.sh` -* omega\_run.sh : run `./src/omega.exe` after sourcing `omega\_env.sh` -* omega\_ctest.sh : run `ctest` after sourcing `omega\_env.sh` +* omega\_build.sh : run `make` command after sourcing `omega_env.sh` +* omega\_run.sh : run `./src/omega.exe` after sourcing `omega_env.sh` +* omega\_ctest.sh : run `ctest` after sourcing `omega_env.sh` Run omega\_build.sh in the build directory to build Omega. @@ -64,11 +68,6 @@ To build Omega for GPU, add OMEGA\_ARCH to one of "CUDA" or "HIP". -DOMEGA_ARCH=CUDA \ ${E3SM_HOME}/components/omega ``` -In some cases, you may want to add OMEGA\_CIME\_MACHINE to specify which -system you intend to use. - -The values of OMEGA\_CIME\_COMPILER and OMEGA\_CIME\_MACHINE are defined in -"${E3SM}/cime\_config/machines/config\_machines.xml". To enable the ctest-based unittest option, include the `OMEGA_BUILD_TEST` option as illustrated below.