diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 70dfa7c434..225ebe49dd 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -5,7 +5,7 @@ import os import re -import subprocess +import systemtest_utils as stu from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.XML.standard_module_setup import * from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files @@ -66,61 +66,18 @@ def _create_config_file(self): cfg_out.write(line) def _run_modify_fsurdat(self): - tool_path = os.path.join(self._ctsm_root, "tools/modify_input_files/fsurdat_modifier") - + tool_path = os.path.join(self._ctsm_root, + 'tools/modify_input_files/fsurdat_modifier') self._case.load_env(reset=True) - conda_env = ". " + self._get_caseroot() + "/.env_mach_specific.sh; " - # Preprend the commands to get the conda environment for python first - conda_env += self._get_conda_env() - # Source the env - try: - subprocess.run( - conda_env + "python3 " + tool_path + " " + self._cfg_file_path, - shell=True, - check=True, - ) - except subprocess.CalledProcessError as error: - print("ERROR while getting the conda environment and/or ") - print("running the fsurdat_modifier tool: ") - print("(1) If your ctsm_pylib environment is out of date or you ") - print("have not created the ctsm_pylib environment, yet, you may ") - print("get past this error by running ./py_env_create ") - print("in your ctsm directory and trying this test again. ") - print("(2) If conda is not available, install and load conda, ") - print("run ./py_env_create, and then try this test again. ") - print("(3) If (1) and (2) are not the issue, then you may be ") - print("getting an error within the fsurdat_modifier tool itself. ") - print("Default error message: ") - print(error.output) - raise - except: - print("ERROR trying to run fsurdat_modifier tool.") - raise + command = f"python3 {tool_path} {self._cfg_file_path}" + stu.run_python_script( + self._get_caseroot(), + "ctsm_pylib", + command, + tool_path, + ) def _modify_user_nl(self): - append_to_user_nl_files( - caseroot=self._get_caseroot(), - component="clm", - contents="fsurdat = '{}'".format(self._fsurdat_out), - ) - - def _get_conda_env(self): - # - # Add specific commands needed on different machines to get conda available - # Use semicolon here since it's OK to fail - # - # Execute the module unload/load when "which conda" fails - # eg on cheyenne - try: - subprocess.run("which conda", shell=True, check=True) - conda_env = " " - except subprocess.CalledProcessError: - # Remove python and add conda to environment for cheyennne - conda_env = "module unload python; module load conda;" - - # Activate the python environment - conda_env += " conda activate ctsm_pylib" - # End above to get to actual command - conda_env += " && " - - return conda_env + append_to_user_nl_files(caseroot = self._get_caseroot(), + component = "clm", + contents = "fsurdat = '{}'".format(self._fsurdat_out)) diff --git a/cime_config/SystemTests/systemtest_utils.py b/cime_config/SystemTests/systemtest_utils.py new file mode 100644 index 0000000000..3c264b32b9 --- /dev/null +++ b/cime_config/SystemTests/systemtest_utils.py @@ -0,0 +1,55 @@ +""" +Reduce code duplication by putting reused functions here. +""" + +import os, subprocess + +def cmds_to_setup_conda(caseroot): + # Add specific commands needed on different machines to get conda available + # Use semicolon here since it's OK to fail + # + conda_setup_commands = ". " + caseroot + "/.env_mach_specific.sh; " + # Execute the module unload/load when "which conda" fails + # eg on cheyenne + try: + subprocess.run( "which conda", shell=True, check=True) + except subprocess.CalledProcessError: + # Remove python and add conda to environment for cheyennne + conda_setup_commands += " module unload python; module load conda;" + + return conda_setup_commands + + +def run_python_script(caseroot, this_conda_env, command, tool_path): + + # Run in the specified conda environment + conda_setup_commands = cmds_to_setup_conda(caseroot) + conda_setup_commands += f" conda run -n {this_conda_env}" + + # Finish with Python script call + command = conda_setup_commands + " " + command + print(f"command: {command}") + + # Run with logfile + tool_name = os.path.split(tool_path)[-1] + try: + with open(tool_name + ".log", "w") as f: + subprocess.run(command, shell=True, check=True, text=True, + stdout=f, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as error: + print("ERROR while getting the conda environment and/or ") + print(f"running the {tool_name} tool: ") + print(f"(1) If your {this_conda_env} environment is out of date or you ") + print(f"have not created the {this_conda_env} environment, yet, you may ") + print("get past this error by running ./py_env_create ") + print("in your ctsm directory and trying this test again. ") + print("(2) If conda is not available, install and load conda, ") + print("run ./py_env_create, and then try this test again. ") + print("(3) If (1) and (2) are not the issue, then you may be ") + print(f"getting an error within {tool_name} itself. ") + print("Default error message: ") + print(error.output) + raise + except: + print(f"ERROR trying to run {tool_name}.") + raise diff --git a/doc/ChangeLog.tmp b/doc/ChangeLog.tmp new file mode 100644 index 0000000000..d70edb368a --- /dev/null +++ b/doc/ChangeLog.tmp @@ -0,0 +1,62 @@ +=============================================================== +Tag name: ctsm5.1.dev131 +Originator(s): samrabin (Sam Rabin,UCAR/TSS) +Date: Mon Jul 17 13:40:04 MDT 2023 +One-line Summary: Resolve some users' errors with Python tests. + +Purpose and description of changes +---------------------------------- + +Fixes two issues some users encountered: +* `conda activate` error in FSURDATMODIFYCTSM +* FileNotFoundError for 15 tests in run_ctsm_py_tests + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): + Fixes #2042: Issue running SystemTests due to "conda activate" error + +Notes of particular relevance for developers: +--------------------------------------------- + +Changes to tests or testing: Fixes for some tests. + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + cheyenne - PASS + + clm_pymods test suite on cheyenne - PASS + + +Other details +------------- + +Pull Requests that document the changes (include PR ids): + https://github.com/ESCOMP/CTSM/issues/2042 + +=============================================================== diff --git a/doc/ChangeSum b/doc/ChangeSum index a3f748b0d0..2ea87aa85f 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev131 samrabin 07/17/2023 Resolve some users' errors with Python tests. ctsm5.1.dev130 glemieux 07/09/2023 FATES parameter file and test definition update ctsm5.1.dev129 erik 06/22/2023 NEON fixes for TOOL and user-mods, add SP for NEON, some history file updates, black refactor for buildlib/buildnml ctsm5.1.dev128 glemieux 06/01/2023 Update FATES tests to double precision diff --git a/python/ctsm/test/test_sys_mesh_modifier.py b/python/ctsm/test/test_sys_mesh_modifier.py index 0889a505ba..0804900b36 100755 --- a/python/ctsm/test/test_sys_mesh_modifier.py +++ b/python/ctsm/test/test_sys_mesh_modifier.py @@ -57,6 +57,7 @@ def setUp(self): metadata_file = os.path.join(self._tempdir, "metadata.nc") configure_path = os.path.join(path_to_cime(), "CIME/scripts/configure") + self._original_wd = os.getcwd() os.chdir(self._tempdir) # cd to tempdir # Run configure to generate .env_mach_specific.sh @@ -111,7 +112,7 @@ def tearDown(self): """ Remove temporary directory """ - os.getcwd() # cd back to the original working directory + os.chdir(self._original_wd) # cd back to the original working directory shutil.rmtree(self._tempdir, ignore_errors=True) def test_allInfo(self):