From 4bf4b2e407d29869601e0775d54a2f0bf3c8e6e8 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 26 Jun 2023 17:13:12 -0600 Subject: [PATCH] FSURDATMODIFYCTSM test now uses `conda run -n` instead of `conda activate`. This is a more robust method and is recommended when conda is being used non-interactively. RXCROPMATURITY already used this method, so I introduced a new module in cime_config/SystemTests, systemtests_utils.py, to share the code. * Resolves #2042 --- cime_config/SystemTests/fsurdatmodifyctsm.py | 69 ++++---------------- cime_config/SystemTests/systemtest_utils.py | 55 ++++++++++++++++ 2 files changed, 68 insertions(+), 56 deletions(-) create mode 100644 cime_config/SystemTests/systemtest_utils.py diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 70dfa7c434..39cd913e3c 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -5,7 +5,7 @@ import os import re -import subprocess +import systemtest_utils as stu from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.XML.standard_module_setup import * from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files @@ -66,61 +66,18 @@ def _create_config_file(self): cfg_out.write(line) def _run_modify_fsurdat(self): - tool_path = os.path.join(self._ctsm_root, "tools/modify_input_files/fsurdat_modifier") - + tool_path = os.path.join(self._ctsm_root, + "tools/modify_input_files/fsurdat_modifier") self._case.load_env(reset=True) - conda_env = ". " + self._get_caseroot() + "/.env_mach_specific.sh; " - # Preprend the commands to get the conda environment for python first - conda_env += self._get_conda_env() - # Source the env - try: - subprocess.run( - conda_env + "python3 " + tool_path + " " + self._cfg_file_path, - shell=True, - check=True, - ) - except subprocess.CalledProcessError as error: - print("ERROR while getting the conda environment and/or ") - print("running the fsurdat_modifier tool: ") - print("(1) If your ctsm_pylib environment is out of date or you ") - print("have not created the ctsm_pylib environment, yet, you may ") - print("get past this error by running ./py_env_create ") - print("in your ctsm directory and trying this test again. ") - print("(2) If conda is not available, install and load conda, ") - print("run ./py_env_create, and then try this test again. ") - print("(3) If (1) and (2) are not the issue, then you may be ") - print("getting an error within the fsurdat_modifier tool itself. ") - print("Default error message: ") - print(error.output) - raise - except: - print("ERROR trying to run fsurdat_modifier tool.") - raise + command = f"python3 {tool_path} {self._cfg_file_path}" + stu.run_python_script( + self._get_caseroot(), + "ctsm_pylib", + command, + tool_path, + ) def _modify_user_nl(self): - append_to_user_nl_files( - caseroot=self._get_caseroot(), - component="clm", - contents="fsurdat = '{}'".format(self._fsurdat_out), - ) - - def _get_conda_env(self): - # - # Add specific commands needed on different machines to get conda available - # Use semicolon here since it's OK to fail - # - # Execute the module unload/load when "which conda" fails - # eg on cheyenne - try: - subprocess.run("which conda", shell=True, check=True) - conda_env = " " - except subprocess.CalledProcessError: - # Remove python and add conda to environment for cheyennne - conda_env = "module unload python; module load conda;" - - # Activate the python environment - conda_env += " conda activate ctsm_pylib" - # End above to get to actual command - conda_env += " && " - - return conda_env + append_to_user_nl_files(caseroot = self._get_caseroot(), + component = "clm", + contents = "fsurdat = '{}'".format(self._fsurdat_out)) diff --git a/cime_config/SystemTests/systemtest_utils.py b/cime_config/SystemTests/systemtest_utils.py new file mode 100644 index 0000000000..3c264b32b9 --- /dev/null +++ b/cime_config/SystemTests/systemtest_utils.py @@ -0,0 +1,55 @@ +""" +Reduce code duplication by putting reused functions here. +""" + +import os, subprocess + +def cmds_to_setup_conda(caseroot): + # Add specific commands needed on different machines to get conda available + # Use semicolon here since it's OK to fail + # + conda_setup_commands = ". " + caseroot + "/.env_mach_specific.sh; " + # Execute the module unload/load when "which conda" fails + # eg on cheyenne + try: + subprocess.run( "which conda", shell=True, check=True) + except subprocess.CalledProcessError: + # Remove python and add conda to environment for cheyennne + conda_setup_commands += " module unload python; module load conda;" + + return conda_setup_commands + + +def run_python_script(caseroot, this_conda_env, command, tool_path): + + # Run in the specified conda environment + conda_setup_commands = cmds_to_setup_conda(caseroot) + conda_setup_commands += f" conda run -n {this_conda_env}" + + # Finish with Python script call + command = conda_setup_commands + " " + command + print(f"command: {command}") + + # Run with logfile + tool_name = os.path.split(tool_path)[-1] + try: + with open(tool_name + ".log", "w") as f: + subprocess.run(command, shell=True, check=True, text=True, + stdout=f, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as error: + print("ERROR while getting the conda environment and/or ") + print(f"running the {tool_name} tool: ") + print(f"(1) If your {this_conda_env} environment is out of date or you ") + print(f"have not created the {this_conda_env} environment, yet, you may ") + print("get past this error by running ./py_env_create ") + print("in your ctsm directory and trying this test again. ") + print("(2) If conda is not available, install and load conda, ") + print("run ./py_env_create, and then try this test again. ") + print("(3) If (1) and (2) are not the issue, then you may be ") + print(f"getting an error within {tool_name} itself. ") + print("Default error message: ") + print(error.output) + raise + except: + print(f"ERROR trying to run {tool_name}.") + raise