Resolve some users' errors with Python tests

cime_config/SystemTests/fsurdatmodifyctsm.py

@@ -5,7 +5,7 @@
 
 import os
 import re
-import subprocess
+import systemtest_utils as stu
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files
@@ -66,61 +66,18 @@ def _create_config_file(self):
                     cfg_out.write(line)
 
     def _run_modify_fsurdat(self):
-        tool_path = os.path.join(self._ctsm_root, "tools/modify_input_files/fsurdat_modifier")
-
+        tool_path = os.path.join(self._ctsm_root,
+                                 'tools/modify_input_files/fsurdat_modifier')
         self._case.load_env(reset=True)
-        conda_env = ". " + self._get_caseroot() + "/.env_mach_specific.sh; "
-        # Preprend the commands to get the conda environment for python first
-        conda_env += self._get_conda_env()
-        # Source the env
-        try:
-            subprocess.run(
-                conda_env + "python3 " + tool_path + " " + self._cfg_file_path,
-                shell=True,
-                check=True,
-            )
-        except subprocess.CalledProcessError as error:
-            print("ERROR while getting the conda environment and/or ")
-            print("running the fsurdat_modifier tool: ")
-            print("(1) If your ctsm_pylib environment is out of date or you ")
-            print("have not created the ctsm_pylib environment, yet, you may ")
-            print("get past this error by running ./py_env_create ")
-            print("in your ctsm directory and trying this test again. ")
-            print("(2) If conda is not available, install and load conda, ")
-            print("run ./py_env_create, and then try this test again. ")
-            print("(3) If (1) and (2) are not the issue, then you may be ")
-            print("getting an error within the fsurdat_modifier tool itself. ")
-            print("Default error message: ")
-            print(error.output)
-            raise
-        except:
-            print("ERROR trying to run fsurdat_modifier tool.")
-            raise
+        command = f"python3 {tool_path} {self._cfg_file_path}"
+        stu.run_python_script(
+                self._get_caseroot(),
+                "ctsm_pylib",
+                command,
+                tool_path,
+                )
 
     def _modify_user_nl(self):
-        append_to_user_nl_files(
-            caseroot=self._get_caseroot(),
-            component="clm",
-            contents="fsurdat = '{}'".format(self._fsurdat_out),
-        )
-
-    def _get_conda_env(self):
-        #
-        # Add specific commands needed on different machines to get conda available
-        # Use semicolon here since it's OK to fail
-        #
-        # Execute the module unload/load when "which conda" fails
-        # eg on cheyenne
-        try:
-            subprocess.run("which conda", shell=True, check=True)
-            conda_env = " "
-        except subprocess.CalledProcessError:
-            # Remove python and add conda to environment for cheyennne
-            conda_env = "module unload python; module load conda;"
-
-        # Activate the python environment
-        conda_env += " conda activate ctsm_pylib"
-        # End above to get to actual command
-        conda_env += " && "
-
-        return conda_env
+        append_to_user_nl_files(caseroot = self._get_caseroot(),
+                                component = "clm",
+                                contents = "fsurdat = '{}'".format(self._fsurdat_out))

cime_config/SystemTests/systemtest_utils.py

@@ -0,0 +1,55 @@
+"""
+Reduce code duplication by putting reused functions here.
+"""
+
+import os, subprocess
+
+def cmds_to_setup_conda(caseroot):
+    # Add specific commands needed on different machines to get conda available
+    # Use semicolon here since it's OK to fail
+    #
+    conda_setup_commands = ". " + caseroot + "/.env_mach_specific.sh; "
+    # Execute the module unload/load when "which conda" fails
+    # eg on cheyenne
+    try:
+        subprocess.run( "which conda", shell=True, check=True)
+    except subprocess.CalledProcessError:
+        # Remove python and add conda to environment for cheyennne
+        conda_setup_commands += " module unload python; module load conda;"
+
+    return conda_setup_commands
+
+
+def run_python_script(caseroot, this_conda_env, command, tool_path):
+
+    # Run in the specified conda environment
+    conda_setup_commands = cmds_to_setup_conda(caseroot)
+    conda_setup_commands += f" conda run -n {this_conda_env}"
+
+    # Finish with Python script call
+    command = conda_setup_commands + " " + command
+    print(f"command: {command}")
+
+    # Run with logfile
+    tool_name = os.path.split(tool_path)[-1]
+    try:
+        with open(tool_name + ".log", "w") as f:
+            subprocess.run(command, shell=True, check=True, text=True,
+                stdout=f, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as error:
+        print("ERROR while getting the conda environment and/or ")
+        print(f"running the {tool_name} tool: ")
+        print(f"(1) If your {this_conda_env} environment is out of date or you ")
+        print(f"have not created the {this_conda_env} environment, yet, you may ")
+        print("get past this error by running ./py_env_create ")
+        print("in your ctsm directory and trying this test again. ")
+        print("(2) If conda is not available, install and load conda, ")
+        print("run ./py_env_create, and then try this test again. ")
+        print("(3) If (1) and (2) are not the issue, then you may be ")
+        print(f"getting an error within {tool_name} itself. ")
+        print("Default error message: ")
+        print(error.output)
+        raise
+    except:
+        print(f"ERROR trying to run {tool_name}.")
+        raise

doc/ChangeLog.tmp

@@ -0,0 +1,62 @@
+===============================================================
+Tag name: ctsm5.1.dev131
+Originator(s): samrabin (Sam Rabin,UCAR/TSS)
+Date: Mon Jul 17 13:40:04 MDT 2023
+One-line Summary: Resolve some users' errors with Python tests.
+
+Purpose and description of changes
+----------------------------------
+
+Fixes two issues some users encountered:
+* `conda activate` error in FSURDATMODIFYCTSM
+* FileNotFoundError for 15 tests in run_ctsm_py_tests
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed or introduced
+------------------------
+
+CTSM issues fixed (include CTSM Issue #):
+    Fixes #2042: Issue running SystemTests due to "conda activate" error
+
+Notes of particular relevance for developers:
+---------------------------------------------
+
+Changes to tests or testing: Fixes for some tests.
+
+
+Testing summary:
+----------------
+
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    cheyenne - PASS
+
+    clm_pymods test suite on cheyenne - PASS
+
+
+Other details
+-------------
+
+Pull Requests that document the changes (include PR ids):
+    https://github.com/ESCOMP/CTSM/issues/2042
+
+===============================================================

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+    ctsm5.1.dev131 samrabin 07/17/2023 Resolve some users' errors with Python tests.
     ctsm5.1.dev130 glemieux 07/09/2023 FATES parameter file and test definition update
     ctsm5.1.dev129     erik 06/22/2023 NEON fixes for TOOL and user-mods, add SP for NEON, some history file updates, black refactor for buildlib/buildnml
     ctsm5.1.dev128 glemieux 06/01/2023 Update FATES tests to double precision

python/ctsm/test/test_sys_mesh_modifier.py

@@ -57,6 +57,7 @@ def setUp(self):
         metadata_file = os.path.join(self._tempdir, "metadata.nc")
         configure_path = os.path.join(path_to_cime(), "CIME/scripts/configure")
 
+        self._original_wd = os.getcwd()
         os.chdir(self._tempdir)  # cd to tempdir
 
         # Run configure to generate .env_mach_specific.sh
@@ -111,7 +112,7 @@ def tearDown(self):
         """
         Remove temporary directory
         """
-        os.getcwd()  # cd back to the original working directory
+        os.chdir(self._original_wd)  # cd back to the original working directory
         shutil.rmtree(self._tempdir, ignore_errors=True)
 
     def test_allInfo(self):