From f9f38b61dd89ed8e79f61f6f6dbe24206303753e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 10 Jan 2024 10:33:03 -0700 Subject: [PATCH 1/7] initial refactoring from refactoring branch --- python/ctsm/site_and_regional/arg_parse.py | 267 +++++++++ python/ctsm/site_and_regional/neon_site.py | 393 ++++++++++++ python/ctsm/site_and_regional/run_neon.py | 658 +-------------------- python/ctsm/test/test_unit_NeonSite.py | 64 ++ python/ctsm/test/test_unit_arg_parse.py | 64 ++ tools/site_and_regional/run_neon | 4 + 6 files changed, 804 insertions(+), 646 deletions(-) create mode 100644 python/ctsm/site_and_regional/arg_parse.py create mode 100755 python/ctsm/site_and_regional/neon_site.py create mode 100755 python/ctsm/test/test_unit_NeonSite.py create mode 100755 python/ctsm/test/test_unit_arg_parse.py diff --git a/python/ctsm/site_and_regional/arg_parse.py b/python/ctsm/site_and_regional/arg_parse.py new file mode 100644 index 0000000000..9025f5dde8 --- /dev/null +++ b/python/ctsm/site_and_regional/arg_parse.py @@ -0,0 +1,267 @@ +""" +Argument parser to use throughout run_neon.py +""" + +import argparse +import datetime +import logging +import os +import sys + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm import add_cime_to_path + +from CIME import build +from CIME.case import Case +from CIME.utils import safe_copy, expect, parse_args_and_handle_standard_logging_options, setup_standard_logging_options, symlink_force + +from ctsm.utils import parse_isoduration + + +def get_parser(args, description, valid_neon_sites): + """ + Get parser object for this script. + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + setup_standard_logging_options(parser) + + parser.print_usage = parser.print_help + + parser.add_argument( + "--neon-sites", + help="4-letter neon site code.", + action="store", + required=False, + choices=valid_neon_sites + ["all"], + dest="neon_sites", + default=["OSBS"], + nargs="+", + ) + + parser.add_argument( + "--base-case", + help=""" + Root Directory of base case build + [default: %(default)s] + """, + action="store", + dest="base_case_root", + type=str, + required=False, + default=None, + ) + + parser.add_argument( + "--output-root", + help=""" + Root output directory of cases + [default: %(default)s] + """, + action="store", + dest="output_root", + type=str, + required=False, + default="CIME_OUTPUT_ROOT as defined in cime", + ) + + parser.add_argument( + "--overwrite", + help=""" + overwrite existing case directories + [default: %(default)s] + """, + action="store_true", + dest="overwrite", + required=False, + default=False, + ) + + parser.add_argument( + "--setup-only", + help=""" + Only setup the requested cases, do not build or run + [default: %(default)s] + """, + action="store_true", + dest="setup_only", + required=False, + default=False, + ) + + parser.add_argument( + "--rerun", + help=""" + If the case exists but does not appear to be complete, restart it. + [default: %(default)s] + """, + action="store_true", + dest="rerun", + required=False, + default=False, + ) + + parser.add_argument( + "--no-batch", + help=""" + Run locally, do not use batch queueing system (if defined for Machine) + [default: %(default)s] + """, + action="store_true", + dest="no_batch", + required=False, + default=False, + ) + + parser.add_argument( + "--run-type", + help=""" + Type of run to do + [default: %(default)s] + """, + choices=["ad", "postad", "transient"], #, "sasu"], + default="transient", + ) + + parser.add_argument( + "--prism", + help=""" + Uses the PRISM reanaylsis precipitation data for the site instead of the NEON data + (only available over Continental US) + """, + action="store_true", + dest="prism", + required=False, + default=False, + ) + + parser.add_argument( + "--experiment", + help=""" + Appends the case name with string for model experiment + """, + action="store", + dest="experiment", + type=str, + required=False, + default=None, + ) + + parser.add_argument( + "--run-length", + help=""" + How long to run (modified ISO 8601 duration) + [default: %(default)s] + """, + required=False, + type=str, + default="0Y", + ) + + #parser.add_argument( + # "--start-date", + # help=""" + # Start date for running CTSM simulation in ISO format. + # [default: %(default)s] + # (currently non-functional) + # """, + # action="store", + # dest="start_date", + # required=False, + # type=datetime.date.fromisoformat, + # default=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), + #) + + #parser.add_argument( + # "--end-date", + # help=""" + # End date for running CTSM simulation in ISO format. + # [default: %(default)s] + # """, + # action="store", + # dest="end_date", + # required=False, + # type=datetime.date.fromisoformat, + # default=datetime.datetime.strptime("2021-01-01", "%Y-%m-%d"), + #) + + parser.add_argument( + "--run-from-postad", + help=""" + For transient runs only - should we start from the postad spinup or finidat? + By default start from finidat, if this flag is used the postad run must be available. + """, + action="store_true", + required=False, + default=False, + ) + parser.add_argument( + "--neon-version", + help=""" + Neon data version to use for this simulation. + [default: use the latest data available] + """, + action="store", + dest="user_version", + required=False, + type=str, + choices=["v1", "v2", "v3"], + ) + + args = parse_args_and_handle_standard_logging_options(args, parser) + + if "all" in args.neon_sites: + neon_sites = valid_neon_sites + else: + neon_sites = args.neon_sites + for site in neon_sites: + if site not in valid_neon_sites: + raise ValueError("Invalid site name {}".format(site)) + + if "CIME_OUTPUT_ROOT" in args.output_root: + args.output_root = None + + if args.run_length == "0Y": + if args.run_type == "ad": + run_length = "100Y" + elif args.run_type == "postad": + run_length = "100Y" + else: + # The transient run length is set by cdeps atm buildnml to + # the last date of the available tower data + # this value is not used + run_length = "4Y" + else: + run_length = args.run_length + + run_length = parse_isoduration(run_length) + base_case_root = None + if args.base_case_root: + base_case_root = os.path.abspath(args.base_case_root) + + # Reduce output level for this script unless --debug or + # --verbose is provided on the command line + if not args.debug and not args.verbose: + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARN) + + return ( + neon_sites, + args.output_root, + args.run_type, + args.experiment, + args.prism, + args.overwrite, + run_length, + base_case_root, + args.run_from_postad, + args.setup_only, + args.no_batch, + args.rerun, + args.user_version, + ) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py new file mode 100755 index 0000000000..873d02fec7 --- /dev/null +++ b/python/ctsm/site_and_regional/neon_site.py @@ -0,0 +1,393 @@ +""" +This module contains the NeonSite class and class functions which are used in run_neon.py +""" + +# Import libraries +import datetime +import glob +import logging +import os +import re +import shutil +import sys +import time + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm import add_cime_to_path + +from CIME import build +from CIME.case import Case +from CIME.utils import safe_copy, expect, symlink_force + +from ctsm.path_utils import path_to_ctsm_root + +logger = logging.getLogger(__name__) + + +class NeonSite: + """ + A class for encapsulating neon sites. + """ + + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): + self.name = name + self.start_year = int(start_year) + self.end_year = int(end_year) + self.start_month = int(start_month) + self.end_month = int(end_month) + self.cesmroot = path_to_ctsm_root() + self.finidat = finidat + + def __str__(self): + return ( + str(self.__class__) + "\n" + "\n".join((str(item) + " = " for item in (self.__dict__))) + ) + + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + """ + Function for building a base_case to clone. + To spend less time on building ctsm for the neon cases, + all the other cases are cloned from this case + + Args: + self: + The NeonSite object + base_root (str): + root of the base_case CIME + res (str): + base_case resolution or gridname + compset (str): + base case compset + overwrite (bool) : + Flag to overwrite the case if exists + """ + print("---- building a base case -------") + self.base_case_root = output_root + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root, self.name) + + logger.info("base_case_name : %s", self.name) + logger.info("user_mods_dir : %s", user_mods_dirs[0]) + + if overwrite and os.path.isdir(case_path): + print("Removing the existing case at: {}".format(case_path)) + shutil.rmtree(case_path) + + with Case(case_path, read_only=False) as case: + if not os.path.isdir(case_path): + print("---- creating a base case -------") + + case.create( + case_path, + cesmroot, + compset, + res, + run_unsupported=True, + answer="r", + output_root=output_root, + user_mods_dirs=user_mods_dirs, + driver="nuopc", + ) + + print("---- base case created ------") + + # --change any config for base_case: + # case.set_value("RUN_TYPE","startup") + print("---- base case setup ------") + case.case_setup() + else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + # reset the case + case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path + + print("---- base case build ------") + print("--- This may take a while and you may see WARNING messages ---") + # always walk through the build process to make sure it's up to date. + initial_time = time.time() + build.case_build(case_path, case=case) + end_time = time.time() + total = end_time - initial_time + print("Time required to building the base case: {} s.".format(total)) + # update case_path to be the full path to the base case + return case_path + + def get_batch_query(self, case): + """ + Function for querying the batch queue query command for a case, depending on the + user's batch system. + + Args: + case: + case object + """ + + if case.get_value("BATCH_SYSTEM") == "none": + return "none" + return case.get_value("batch_query") + + def run_case( + self, + base_case_root, + run_type, + prism, + run_length, + user_version, + overwrite=False, + setup_only=False, + no_batch=False, + rerun=False, + experiment=False, + ): + """ + Run case. + + Args: + self + base_case_root: str, opt + file path of base case + run_type: str, opt + transient, post_ad, or ad case, default transient + prism: bool, opt + if True, use PRISM precipitation, default False + run_length: str, opt + length of run, default '4Y' + user_version: str, opt + default 'latest' + overwrite: bool, opt + default False + setup_only: bool, opt + default False; if True, set up but do not run case + no_batch: bool, opt + default False + rerun: bool, opt + default False + experiment: str, opt + name of experiment, default False + """ + user_mods_dirs = [ + os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) + ] + expect( + os.path.isdir(base_case_root), + "Error base case does not exist in {}".format(base_case_root), + ) + # -- if user gives a version: + if user_version: + version = user_version + else: + version = "latest" + + print("using this version:", version) + + if experiment is not None: + self.name = self.name + "." + experiment + case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) + + rundir = None + if os.path.isdir(case_root): + if overwrite: + print("---- removing the existing case -------") + shutil.rmtree(case_root) + elif rerun: + with Case(case_root, read_only=False) as case: + rundir = case.get_value("RUNDIR") + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): + print("Case {} appears to be complete, not rerunning.".format(case_root)) + elif not setup_only: + print("Resubmitting case {}".format(case_root)) + case.submit(no_batch=no_batch) + print("-----------------------------------") + print("Successfully submitted case!") + batch_query = self.get_batch_query(case) + if batch_query != "none": + print(f"Use {batch_query} to check its run status") + return + else: + logger.warning("Case already exists in %s, not overwritting", case_root) + return + + if run_type == "postad": + adcase_root = case_root.replace(".postad", ".ad") + if not os.path.isdir(adcase_root): + logger.warning("postad requested but no ad case found in %s", adcase_root) + return + + if not os.path.isdir(case_root): + # read_only = False should not be required here + with Case(base_case_root, read_only=False) as basecase: + print("---- cloning the base case in {}".format(case_root)) + # + # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although + # it causes some of the user_nl_* files to have duplicated inputs. It also ensures + # that the shell_commands file is copied, as well as taking care of the DATM inputs. + # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 + # + basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) + + with Case(case_root, read_only=False) as case: + if run_type != "transient": + # in order to avoid the complication of leap years, + # we always set the run_length in units of days. + case.set_value("STOP_OPTION", "ndays") + case.set_value("REST_OPTION", "end") + case.set_value("CONTINUE_RUN", False) + case.set_value("NEONVERSION", version) + if prism: + case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") + + if run_type == "ad": + case.set_value("CLM_FORCE_COLDSTART", "on") + case.set_value("CLM_ACCELERATED_SPINUP", "on") + case.set_value("RUN_REFDATE", "0018-01-01") + case.set_value("RUN_STARTDATE", "0018-01-01") + case.set_value("RESUBMIT", 1) + case.set_value("STOP_N", run_length) + + else: + case.set_value("CLM_FORCE_COLDSTART", "off") + case.set_value("CLM_ACCELERATED_SPINUP", "off") + case.set_value("RUN_TYPE", "hybrid") + + if run_type == "postad": + self.set_ref_case(case) + case.set_value("STOP_N", run_length) + + # For transient cases STOP will be set in the user_mod_directory + if run_type == "transient": + if self.finidat: + case.set_value("RUN_TYPE", "startup") + else: + if not self.set_ref_case(case): + return + case.set_value("CALENDAR", "GREGORIAN") + case.set_value("RESUBMIT", 0) + case.set_value("STOP_OPTION", "nmonths") + + if not rundir: + rundir = case.get_value("RUNDIR") + + self.modify_user_nl(case_root, run_type, rundir) + + case.create_namelists() + # explicitly run check_input_data + case.check_all_input_data() + if not setup_only: + case.submit(no_batch=no_batch) + print("-----------------------------------") + print("Successfully submitted case!") + batch_query = self.get_batch_query(case) + if batch_query != "none": + print(f"Use {batch_query} to check its run status") + + def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ + rundir = case.get_value("RUNDIR") + case_root = case.get_value("CASEROOT") + if case_root.endswith(".postad"): + ref_case_root = case_root.replace(".postad", ".ad") + root = ".ad" + else: + ref_case_root = case_root.replace(".transient", ".postad") + root = ".postad" + if not os.path.isdir(ref_case_root): + logger.warning( + "ERROR: spinup must be completed first, could not find directory %s", ref_case_root + ) + return False + + with Case(ref_case_root) as refcase: + refrundir = refcase.get_value("RUNDIR") + case.set_value("RUN_REFDIR", refrundir) + case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): + m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) + if m_searched: + refdate = m_searched.group(1) + symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) + logger.info("Found refdate of %s", refdate) + if not refdate: + logger.warning("Could not find refcase for %s", case_root) + return False + + for rpfile in glob.iglob(refrundir + "/rpointer*"): + safe_copy(rpfile, rundir) + if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( + os.path.join(refrundir, "inputdata") + ): + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) + + case.set_value("RUN_REFDATE", refdate) + if case_root.endswith(".postad"): + case.set_value("RUN_STARTDATE", refdate) + # NOTE: if start options are set, RUN_STARTDATE should be modified here + return True + + def modify_user_nl(self, case_root, run_type, rundir): + """ + Modify user namelist. If transient, include finidat in user_nl; + Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. + """ + user_nl_fname = os.path.join(case_root, "user_nl_clm") + user_nl_lines = None + if run_type == "transient": + if self.finidat: + user_nl_lines = [ + "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) + ] + else: + user_nl_lines = [ + "hist_fincl2 = ''", + "hist_mfilt = 20", + "hist_nhtfrq = -8760", + "hist_empty_htapes = .true.", + """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', + 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'""", + ] + + if user_nl_lines: + with open(user_nl_fname, "a") as nl_file: + for line in user_nl_lines: + nl_file.write("{}\n".format(line)) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index a69dc0bdb0..f7ed477cf1 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -47,670 +47,36 @@ # - [ ] Matrix spin-up if (SASU) Eric merged it in # - [ ] Make sure both AD and SASU are not on at the same time -# - [ ] Make sure CIME and other dependencies is checked out. +# - [ ] Make sure CIME and other dependencies are checked out. # Import libraries -import argparse -import datetime import glob import logging import os -import re -import shutil import sys -import time import pandas as pd -from standard_script_setup import * - # Get the ctsm util tools and then the cime tools. _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) +from ctsm.path_utils import path_to_ctsm_root +from ctsm.download_utils import download_file +from ctsm.site_and_regional.arg_parse import get_parser +from ctsm.site_and_regional.neon_site import NeonSite + from ctsm import add_cime_to_path from CIME import build from CIME.case import Case from CIME.utils import safe_copy, expect, symlink_force -from ctsm.path_utils import path_to_ctsm_root -from ctsm.utils import parse_isoduration -from ctsm.download_utils import download_file - from standard_script_setup import * logger = logging.getLogger(__name__) -def get_parser(args, description, valid_neon_sites): - """ - Get parser object for this script. - """ - parser = argparse.ArgumentParser( - description=description, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - CIME.utils.setup_standard_logging_options(parser) - - parser.print_usage = parser.print_help - - parser.add_argument( - "--neon-sites", - help="4-letter neon site code.", - action="store", - required=False, - choices=valid_neon_sites + ["all"], - dest="neon_sites", - default=["OSBS"], - nargs="+", - ) - - parser.add_argument( - "--base-case", - help=""" - Root Directory of base case build - [default: %(default)s] - """, - action="store", - dest="base_case_root", - type=str, - required=False, - default=None, - ) - - parser.add_argument( - "--output-root", - help=""" - Root output directory of cases - [default: %(default)s] - """, - action="store", - dest="output_root", - type=str, - required=False, - default="CIME_OUTPUT_ROOT as defined in cime", - ) - - parser.add_argument( - "--overwrite", - help=""" - overwrite existing case directories - [default: %(default)s] - """, - action="store_true", - dest="overwrite", - required=False, - default=False, - ) - - parser.add_argument( - "--setup-only", - help=""" - Only setup the requested cases, do not build or run - [default: %(default)s] - """, - action="store_true", - dest="setup_only", - required=False, - default=False, - ) - - parser.add_argument( - "--rerun", - help=""" - If the case exists but does not appear to be complete, restart it. - [default: %(default)s] - """, - action="store_true", - dest="rerun", - required=False, - default=False, - ) - - parser.add_argument( - "--no-batch", - help=""" - Run locally, do not use batch queueing system (if defined for Machine) - [default: %(default)s] - """, - action="store_true", - dest="no_batch", - required=False, - default=False, - ) - - parser.add_argument( - "--run-type", - help=""" - Type of run to do - [default: %(default)s] - """, - choices=["ad", "postad", "transient", "sasu"], - default="transient", - ) - - parser.add_argument( - "--prism", - help=""" - Uses the PRISM reanaylsis precipitation data for the site instead of the NEON data - (only available over Continental US) - """, - action="store_true", - dest="prism", - required=False, - default=False, - ) - - parser.add_argument( - "--experiment", - help=""" - Appends the case name with string for model experiment - """, - action="store", - dest="experiment", - type=str, - required=False, - default=None, - ) - - parser.add_argument( - "--run-length", - help=""" - How long to run (modified ISO 8601 duration) - [default: %(default)s] - """, - required=False, - type=str, - default="0Y", - ) - - parser.add_argument( - "--start-date", - help=""" - Start date for running CTSM simulation in ISO format. - [default: %(default)s] - (currently non-functional) - """, - action="store", - dest="start_date", - required=False, - type=datetime.date.fromisoformat, - default=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), - ) - - parser.add_argument( - "--end-date", - help=""" - End date for running CTSM simulation in ISO format. - [default: %(default)s] - """, - action="store", - dest="end_date", - required=False, - type=datetime.date.fromisoformat, - default=datetime.datetime.strptime("2021-01-01", "%Y-%m-%d"), - ) - - parser.add_argument( - "--run-from-postad", - help=""" - For transient runs only - should we start from the postad spinup or finidat? - By default start from finidat, if this flag is used the postad run must be available. - """, - action="store_true", - required=False, - default=False, - ) - parser.add_argument( - "--neon-version", - help=""" - Neon data version to use for this simulation. - [default: use the latest data available] - """, - action="store", - dest="user_version", - required=False, - type=str, - choices=["v1", "v2", "v3"], - ) - - args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - - if "all" in args.neon_sites: - neon_sites = valid_neon_sites - else: - neon_sites = args.neon_sites - for site in neon_sites: - if site not in valid_neon_sites: - raise ValueError("Invalid site name {}".format(site)) - - if "CIME_OUTPUT_ROOT" in args.output_root: - args.output_root = None - - if args.run_length == "0Y": - if args.run_type == "ad": - run_length = "100Y" - elif args.run_type == "postad": - run_length = "100Y" - else: - # The transient run length is set by cdeps atm buildnml to - # the last date of the available tower data - # this value is not used - run_length = "4Y" - else: - run_length = args.run_length - - run_length = parse_isoduration(run_length) - base_case_root = None - if args.base_case_root: - base_case_root = os.path.abspath(args.base_case_root) - - # Reduce output level for this script unless --debug or - # --verbose is provided on the command line - if not args.debug and not args.verbose: - root_logger = logging.getLogger() - root_logger.setLevel(logging.WARN) - - return ( - neon_sites, - args.output_root, - args.run_type, - args.experiment, - args.prism, - args.overwrite, - run_length, - base_case_root, - args.run_from_postad, - args.setup_only, - args.no_batch, - args.rerun, - args.user_version, - ) - - -class NeonSite: - """ - A class for encapsulating neon sites. - - ... - - Attributes - ---------- - - Methods - ------- - """ - - def __init__(self, name, start_year, end_year, start_month, end_month, finidat): - self.name = name - self.start_year = int(start_year) - self.end_year = int(end_year) - self.start_month = int(start_month) - self.end_month = int(end_month) - self.cesmroot = path_to_ctsm_root() - self.finidat = finidat - - def __str__(self): - return str(self.__class__) + "\n" + "\n".join((str(item) + " = " for item in self.__dict__)) - - def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False - ): - """ - Function for building a base_case to clone. - To spend less time on building ctsm for the neon cases, - all the other cases are cloned from this case - - Args: - self: - The NeonSite object - base_root (str): - root of the base_case CIME - res (str): - base_case resolution or gridname - compset (str): - base case compset - overwrite (bool) : - Flag to overwrite the case if exists - """ - print("---- building a base case -------") - self.base_case_root = output_root - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] - if not output_root: - output_root = os.getcwd() - case_path = os.path.join(output_root, self.name) - - logger.info("base_case_name : %s", self.name) - logger.info("user_mods_dir : %s", user_mods_dirs[0]) - - if overwrite and os.path.isdir(case_path): - print("Removing the existing case at: {}".format(case_path)) - shutil.rmtree(case_path) - - with Case(case_path, read_only=False) as case: - if not os.path.isdir(case_path): - print("---- creating a base case -------") - - case.create( - case_path, - cesmroot, - compset, - res, - run_unsupported=True, - answer="r", - output_root=output_root, - user_mods_dirs=user_mods_dirs, - driver="nuopc", - ) - - print("---- base case created ------") - - # --change any config for base_case: - # case.set_value("RUN_TYPE","startup") - print("---- base case setup ------") - case.case_setup() - else: - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - "Existing base case is a historical type and should " - + "not be -- rerun with the --overwrite option", - ) - else: - expect( - match is not None, - "Existing base case should be a historical type and " - + "is not -- rerun with the --overwrite option", - ) - # reset the case - case.case_setup(reset=True) - case_path = case.get_value("CASEROOT") - - if setup_only: - return case_path - - print("---- base case build ------") - print("--- This may take a while and you may see WARNING messages ---") - # always walk through the build process to make sure it's up to date. - t_0 = time.time() - build.case_build(case_path, case=case) - t_1 = time.time() - total = t_1 - t_0 - print("Time required to building the base case: {} s.".format(total)) - # update case_path to be the full path to the base case - return case_path - - def diff_month(self): - """ - Determine difference between two dates in months - """ - d_1 = datetime.datetime(self.end_year, self.end_month, 1) - d_2 = datetime.datetime(self.start_year, self.start_month, 1) - return (d_1.year - d_2.year) * 12 + d_1.month - d_2.month - - def run_case( - self, - base_case_root, - run_type, - prism, - run_length, - user_version, - overwrite=False, - setup_only=False, - no_batch=False, - rerun=False, - experiment=False, - ): - """ - Run case. - - Args: - self - base_case_root: str, opt - file path of base case - run_type: str, opt - transient, post_ad, or ad case, default transient - prism: bool, opt - if True, use PRISM precipitation, default False - run_length: str, opt - length of run, default '4Y' - user_version: str, opt - default 'latest' - overwrite: bool, opt - default False - setup_only: bool, opt - default False; if True, set up but do not run case - no_batch: bool, opt - default False - rerun: bool, opt - default False - experiment: str, opt - name of experiment, default False - """ - user_mods_dirs = [ - os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) - ] - expect( - os.path.isdir(base_case_root), - "Error base case does not exist in {}".format(base_case_root), - ) - # -- if user gives a version: - if user_version: - version = user_version - else: - version = "latest" - - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - "Existing base case is a historical type and " - + "should not be -- rerun with the --overwrite option", - ) - else: - expect( - match is not None, - "Existing base case should be a historical type " - + "and is not -- rerun with the --overwrite option", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting.", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - mon = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if mon: - refdate = mon.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - - def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", - "hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', " - + "'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', " - + "'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'", - ] - - if user_nl_lines: - with open(user_nl_fname, "a") as f_d: - for line in user_nl_lines: - f_d.write("{}\n".format(line)) - - -def get_batch_query(case): - """ - Function for querying the batch queue query command for a case, depending on the - user's batch system. - - Args: - case: - case object - """ - - if case.get_value("BATCH_SYSTEM") == "none": - return "none" - return case.get_value("batch_query") - - def check_neon_listing(valid_neon_sites): """ A function to download and parse neon listing file. @@ -742,19 +108,19 @@ def parse_neon_listing(listing_file, valid_neon_sites): available_list = [] - d_f = pd.read_csv(listing_file) + listing_df = pd.read_csv(listing_file) # check for finidat files for transient run - finidatlist = d_f[d_f["object"].str.contains("lnd/ctsm")] + finidatlist = listing_df[listing_df["object"].str.contains("lnd/ctsm")] # -- filter lines with atm/cdep - d_f = d_f[d_f["object"].str.contains("atm/cdeps/")] + listing_df = listing_df[listing_df["object"].str.contains("atm/cdeps/")] # -- split the object str to extract site name - d_f = d_f["object"].str.split("/", expand=True) + listing_df = listing_df["object"].str.split("/", expand=True) # -- groupby site name - grouped_df = d_f.groupby(8) + grouped_df = listing_df.groupby(8) for key, _ in grouped_df: # -- check if it is a valid neon site if any(key in x for x in valid_neon_sites): @@ -787,7 +153,7 @@ def parse_neon_listing(listing_file, valid_neon_sites): start_month = tmp_df2[1].iloc[0] end_month = tmp_df2[1].iloc[-1] - logger.debug("Valid neon site found: %s", site_name) + logger.debug("Valid neon site %s found!", site_name) logger.debug("File version %s", latest_version) logger.debug("start_year=%s", start_year) logger.debug("end_year=%s", end_year) diff --git a/python/ctsm/test/test_unit_NeonSite.py b/python/ctsm/test/test_unit_NeonSite.py new file mode 100755 index 0000000000..2e6c2650e5 --- /dev/null +++ b/python/ctsm/test/test_unit_NeonSite.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Unit tests for NeonSite + +You can run this by: + python -m unittest test_unit_NeonSite.py +""" + +import unittest +import tempfile +import shutil +import os +import sys + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.NeonSite import FUNCTION_NAME + +# pylint: disable=invalid-name + + +class TestNeonSite(unittest.TestCase): + """ + Basic class for testing NeonSite.py. + """ + + def setUp(self): + """ + Make /_tempdir for use by these tests. + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_function(self): + """ + Test that NeonSite class is working properly... + """ + #valid_neon_sites = ["ABBY", "BART"] + #previous_dir = os.getcwd() + #os.chdir(self._tempdir) # cd to tempdir + #available_list = check_neon_listing(valid_neon_sites) + #self.assertEqual( + # available_list[0].name, "ABBY", "available list of actual sites not as expected" + #) + #self.assertEqual( + # available_list[1].name, "BART", "available list of actual sites not as expected" + #) + # change to previous dir once listing.csv file is created in tempdir and test complete + #os.chdir(previous_dir) + continue + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_arg_parse.py b/python/ctsm/test/test_unit_arg_parse.py new file mode 100755 index 0000000000..ab4fc2adb8 --- /dev/null +++ b/python/ctsm/test/test_unit_arg_parse.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Unit tests for arg_parse + +You can run this by: + python -m unittest test_unit_arg_parse.py +""" + +import unittest +import tempfile +import shutil +import os +import sys + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.arg_parse import FUNCTION_NAME + +# pylint: disable=invalid-name + + +class Testarg_parse(unittest.TestCase): + """ + Basic class for testing arg_parse.py. + """ + + def setUp(self): + """ + Make /_tempdir for use by these tests. + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_function(self): + """ + Test that arg_parse is working properly... + """ + #valid_neon_sites = ["ABBY", "BART"] + #previous_dir = os.getcwd() + #os.chdir(self._tempdir) # cd to tempdir + #available_list = check_neon_listing(valid_neon_sites) + #self.assertEqual( + # available_list[0].name, "ABBY", "available list of actual sites not as expected" + #) + #self.assertEqual( + # available_list[1].name, "BART", "available list of actual sites not as expected" + #) + # change to previous dir once listing.csv file is created in tempdir and test complete + #os.chdir(previous_dir) + continue + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon index ad930f50e3..d2bdfbe786 100755 --- a/tools/site_and_regional/run_neon +++ b/tools/site_and_regional/run_neon @@ -34,6 +34,7 @@ conda activate ctsm_pylib import os import sys +import argparse # -- add python/ctsm to path _CTSM_PYTHON = os.path.join( @@ -41,6 +42,9 @@ _CTSM_PYTHON = os.path.join( ) sys.path.insert(1, _CTSM_PYTHON) +from ctsm.site_and_regional.neon_site import NeonSite +from ctsm.site_and_regional.arg_parse import get_parser + from ctsm.site_and_regional.run_neon import main if __name__ == "__main__": From 4a8c4995c7578c82a680b52f54fc785d4138212d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 10 Jan 2024 14:10:33 -0700 Subject: [PATCH 2/7] a few more refactoring updates, formatting & testing --- python/ctsm/site_and_regional/arg_parse.py | 3 +- python/ctsm/site_and_regional/neon_site.py | 3 +- python/ctsm/test/test_unit_NeonSite.py | 16 +++++++++-- python/ctsm/test/test_unit_arg_parse.py | 33 +++++++++++++--------- tools/site_and_regional/run_neon | 3 -- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/python/ctsm/site_and_regional/arg_parse.py b/python/ctsm/site_and_regional/arg_parse.py index 9025f5dde8..f61a262911 100644 --- a/python/ctsm/site_and_regional/arg_parse.py +++ b/python/ctsm/site_and_regional/arg_parse.py @@ -13,13 +13,12 @@ sys.path.insert(1, _CTSM_PYTHON) from ctsm import add_cime_to_path +from ctsm.utils import parse_isoduration from CIME import build from CIME.case import Case from CIME.utils import safe_copy, expect, parse_args_and_handle_standard_logging_options, setup_standard_logging_options, symlink_force -from ctsm.utils import parse_isoduration - def get_parser(args, description, valid_neon_sites): """ diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 873d02fec7..c3fe6e2c68 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -17,13 +17,12 @@ sys.path.insert(1, _CTSM_PYTHON) from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root from CIME import build from CIME.case import Case from CIME.utils import safe_copy, expect, symlink_force -from ctsm.path_utils import path_to_ctsm_root - logger = logging.getLogger(__name__) diff --git a/python/ctsm/test/test_unit_NeonSite.py b/python/ctsm/test/test_unit_NeonSite.py index 2e6c2650e5..3832bf8537 100755 --- a/python/ctsm/test/test_unit_NeonSite.py +++ b/python/ctsm/test/test_unit_NeonSite.py @@ -18,7 +18,7 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.site_and_regional.NeonSite import FUNCTION_NAME +from ctsm.site_and_regional.neon_site import NeonSite # pylint: disable=invalid-name @@ -40,7 +40,7 @@ def tearDown(self): """ shutil.rmtree(self._tempdir, ignore_errors=True) - def test_function(self): + def test_build_base_case(self): """ Test that NeonSite class is working properly... """ @@ -57,6 +57,18 @@ def test_function(self): # change to previous dir once listing.csv file is created in tempdir and test complete #os.chdir(previous_dir) continue + def test_get_batch_query(self): + """ + """ + continue + def test_run_case(self): + """ + """ + continue + def test_modify_user_nl(self): + """ + """ + continue if __name__ == "__main__": diff --git a/python/ctsm/test/test_unit_arg_parse.py b/python/ctsm/test/test_unit_arg_parse.py index ab4fc2adb8..f7cf7a97ff 100755 --- a/python/ctsm/test/test_unit_arg_parse.py +++ b/python/ctsm/test/test_unit_arg_parse.py @@ -11,6 +11,7 @@ import shutil import os import sys +import glob # -- add python/ctsm to path (needed if we want to run the test stand-alone) _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) @@ -18,7 +19,8 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.site_and_regional.arg_parse import FUNCTION_NAME +from ctsm.site_and_regional.arg_parse import get_parser +from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name @@ -44,19 +46,22 @@ def test_function(self): """ Test that arg_parse is working properly... """ - #valid_neon_sites = ["ABBY", "BART"] - #previous_dir = os.getcwd() - #os.chdir(self._tempdir) # cd to tempdir - #available_list = check_neon_listing(valid_neon_sites) - #self.assertEqual( - # available_list[0].name, "ABBY", "available list of actual sites not as expected" - #) - #self.assertEqual( - # available_list[1].name, "BART", "available list of actual sites not as expected" - #) - # change to previous dir once listing.csv file is created in tempdir and test complete - #os.chdir(previous_dir) - continue + sys.argv = ["--neon-sites ['ABBY']"] + #arguments= ["--neon-sites", "ABBY"] #, "--experiment 'test'", '--overwrite False', '--setup-only True', '--rerun False', '--run-type ad', '--experiment test'] + description='' + cesmroot = path_to_ctsm_root() + valid_neon_sites = glob.glob( + os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*") + ) + valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) + parsed_arguments = get_parser(sys.argv, description, valid_neon_sites) + + print(parsed_arguments) + self.assertEqual( + parsed_arguments[0], "ABBY", "arguments not processed as expected" + ) + # TODO: Still need to figure out correct formatting to get argument recognized properly! + # TODO: Also it might be useful to add in a number of fake arguments to check that they all work... if __name__ == "__main__": diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon index d2bdfbe786..e20189a374 100755 --- a/tools/site_and_regional/run_neon +++ b/tools/site_and_regional/run_neon @@ -42,9 +42,6 @@ _CTSM_PYTHON = os.path.join( ) sys.path.insert(1, _CTSM_PYTHON) -from ctsm.site_and_regional.neon_site import NeonSite -from ctsm.site_and_regional.arg_parse import get_parser - from ctsm.site_and_regional.run_neon import main if __name__ == "__main__": From d3bffbe824c05ff919f80b1ece6d3e641696ff12 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 11 Jan 2024 11:12:56 -0700 Subject: [PATCH 3/7] included new tests for NeonSite class --- python/ctsm/site_and_regional/neon_site.py | 1 + python/ctsm/test/test_unit_NeonSite.py | 95 ++++++++++++++++++---- tools/site_and_regional/run_neon | 1 - 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index c3fe6e2c68..2e92e7f04b 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -390,3 +390,4 @@ def modify_user_nl(self, case_root, run_type, rundir): with open(user_nl_fname, "a") as nl_file: for line in user_nl_lines: nl_file.write("{}\n".format(line)) + diff --git a/python/ctsm/test/test_unit_NeonSite.py b/python/ctsm/test/test_unit_NeonSite.py index 3832bf8537..1520a58426 100755 --- a/python/ctsm/test/test_unit_NeonSite.py +++ b/python/ctsm/test/test_unit_NeonSite.py @@ -10,6 +10,7 @@ import tempfile import shutil import os +import glob import sys # -- add python/ctsm to path (needed if we want to run the test stand-alone) @@ -42,33 +43,91 @@ def tearDown(self): def test_build_base_case(self): """ - Test that NeonSite class is working properly... + Test that NeonSite class' build_base_case is working properly... """ - #valid_neon_sites = ["ABBY", "BART"] - #previous_dir = os.getcwd() - #os.chdir(self._tempdir) # cd to tempdir - #available_list = check_neon_listing(valid_neon_sites) - #self.assertEqual( - # available_list[0].name, "ABBY", "available list of actual sites not as expected" - #) - #self.assertEqual( - # available_list[1].name, "BART", "available list of actual sites not as expected" - #) - # change to previous dir once listing.csv file is created in tempdir and test complete - #os.chdir(previous_dir) - continue + #neonsite = NeonSite(ADD SOME PARAMETERS) + #neonsite.build_base_case(ARGUMENTS) # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? + #CHECK IF ACTS AS EXPECTED + #continue + def test_get_batch_query(self): """ + Test that NeonSite class' get_batch_query is working properly... """ - continue + #neonsite = NeonSite(ADD SOME PARAMETERS) + #neonsite.get_batch_query(ARGUMENTS) + #CHECK IF ACTS AS EXPECTED + #continue + # ALSO DOESN'T SEEM THE MOST REASONABLE TO TEST + def test_run_case(self): """ + Test that NeonSite class' run_case is working properly... """ - continue - def test_modify_user_nl(self): + #neonsite = NeonSite(ADD SOME PARAMETERS) + #neonsite.run_case(ARGUMENTS) # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? + #CHECK IF ACTS AS EXPECTED + #continue + + def test_modify_user_nl_transient(self): """ + Test that NeonSite class' modify_user_nl is correctly adding lines to namelist for transient cases """ - continue + # NeonSite parameters: + name = 'ABBY' + start_year = 2020 + end_year = 2021 + start_month = 1 + end_month = 12 + #finidat = None + finidat = 'dummy_finidat' + + # modify_user_nl parameters: + case_root = self._tempdir + run_type = 'transient' + rundir = '' + + # update namelist + neonsite = NeonSite(name, start_year, end_year, start_month, end_month, finidat) + modified_neonsite = neonsite.modify_user_nl(case_root, run_type, rundir) + + # gather file contents for test + new_nl_file = open(glob.glob(case_root+'/*')[0], "r") + lines_read = new_nl_file.readlines()[0] + new_nl_file.close() + + # assertion + self.assertEqual(lines_read, "finidat = '/inputdata/lnd/ctsm/initdata/dummy_finidat'\n", 'transient case has unexpected nl') + + def test_modify_user_nl_ad(self): + """ + Test that NeonSite class' modify_user_nl is correctly adding lines to namelist for ad cases + """ + # NeonSite parameters: + name = 'ABBY' + start_year = 2020 + end_year = 2021 + start_month = 1 + end_month = 12 + #finidat = None + finidat = 'dummy_finidat' + + # modify_user_nl parameters: + case_root = self._tempdir + run_type = 'ad' + rundir = '' + + # update namelist + neonsite = NeonSite(name, start_year, end_year, start_month, end_month, finidat) + modified_neonsite = neonsite.modify_user_nl(case_root, run_type, rundir) + + # gather file contents for test + new_nl_file = open(glob.glob(case_root+'/*')[0], "r") + lines_read = new_nl_file.readlines()[1] + new_nl_file.close() + + # assertion + self.assertEqual(lines_read, "hist_mfilt = 20\n", 'ad case has unexpected nl') if __name__ == "__main__": diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon index e20189a374..ad930f50e3 100755 --- a/tools/site_and_regional/run_neon +++ b/tools/site_and_regional/run_neon @@ -34,7 +34,6 @@ conda activate ctsm_pylib import os import sys -import argparse # -- add python/ctsm to path _CTSM_PYTHON = os.path.join( From 88372f2b9fc1a3947f0b743268f93f51f6aff4d4 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 11 Jan 2024 13:39:06 -0700 Subject: [PATCH 4/7] black/pylint --- python/ctsm/site_and_regional/arg_parse.py | 38 +-------- python/ctsm/site_and_regional/neon_site.py | 3 +- python/ctsm/site_and_regional/run_neon.py | 10 +-- python/ctsm/test/test_unit_arg_parse.py | 11 ++- ...nit_NeonSite.py => test_unit_neon_site.py} | 80 ++++++++++--------- tools/site_and_regional/run_neon | 1 + 6 files changed, 59 insertions(+), 84 deletions(-) rename python/ctsm/test/{test_unit_NeonSite.py => test_unit_neon_site.py} (54%) diff --git a/python/ctsm/site_and_regional/arg_parse.py b/python/ctsm/site_and_regional/arg_parse.py index f61a262911..f45fef041c 100644 --- a/python/ctsm/site_and_regional/arg_parse.py +++ b/python/ctsm/site_and_regional/arg_parse.py @@ -3,7 +3,6 @@ """ import argparse -import datetime import logging import os import sys @@ -12,13 +11,11 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) +#pylint: disable=wrong-import-position, import-error, unused-import from ctsm import add_cime_to_path from ctsm.utils import parse_isoduration - -from CIME import build -from CIME.case import Case -from CIME.utils import safe_copy, expect, parse_args_and_handle_standard_logging_options, setup_standard_logging_options, symlink_force - +from CIME.utils import parse_args_and_handle_standard_logging_options +from CIME.utils import setup_standard_logging_options def get_parser(args, description, valid_neon_sites): """ @@ -123,7 +120,7 @@ def get_parser(args, description, valid_neon_sites): Type of run to do [default: %(default)s] """, - choices=["ad", "postad", "transient"], #, "sasu"], + choices=["ad", "postad", "transient"], # , "sasu"], default="transient", ) @@ -162,33 +159,6 @@ def get_parser(args, description, valid_neon_sites): default="0Y", ) - #parser.add_argument( - # "--start-date", - # help=""" - # Start date for running CTSM simulation in ISO format. - # [default: %(default)s] - # (currently non-functional) - # """, - # action="store", - # dest="start_date", - # required=False, - # type=datetime.date.fromisoformat, - # default=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), - #) - - #parser.add_argument( - # "--end-date", - # help=""" - # End date for running CTSM simulation in ISO format. - # [default: %(default)s] - # """, - # action="store", - # dest="end_date", - # required=False, - # type=datetime.date.fromisoformat, - # default=datetime.datetime.strptime("2021-01-01", "%Y-%m-%d"), - #) - parser.add_argument( "--run-from-postad", help=""" diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 2e92e7f04b..8881ef5867 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -3,7 +3,6 @@ """ # Import libraries -import datetime import glob import logging import os @@ -16,6 +15,7 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) +#pylint: disable=wrong-import-position, import-error, unused-import from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root @@ -390,4 +390,3 @@ def modify_user_nl(self, case_root, run_type, rundir): with open(user_nl_fname, "a") as nl_file: for line in user_nl_lines: nl_file.write("{}\n".format(line)) - diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index f7ed477cf1..3802133a64 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -61,18 +61,16 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) +# pylint: disable=wrong-import-position from ctsm.path_utils import path_to_ctsm_root from ctsm.download_utils import download_file from ctsm.site_and_regional.arg_parse import get_parser from ctsm.site_and_regional.neon_site import NeonSite -from ctsm import add_cime_to_path - -from CIME import build -from CIME.case import Case -from CIME.utils import safe_copy, expect, symlink_force - +# pylint: disable=import-error from standard_script_setup import * +# the above import is need to set CIMEROOT environment variable +# TODO: figure out what specifically needs to be imported (not '*') logger = logging.getLogger(__name__) diff --git a/python/ctsm/test/test_unit_arg_parse.py b/python/ctsm/test/test_unit_arg_parse.py index f7cf7a97ff..1da6b63f77 100755 --- a/python/ctsm/test/test_unit_arg_parse.py +++ b/python/ctsm/test/test_unit_arg_parse.py @@ -47,8 +47,9 @@ def test_function(self): Test that arg_parse is working properly... """ sys.argv = ["--neon-sites ['ABBY']"] - #arguments= ["--neon-sites", "ABBY"] #, "--experiment 'test'", '--overwrite False', '--setup-only True', '--rerun False', '--run-type ad', '--experiment test'] - description='' + # arguments= ["--neon-sites", "ABBY"] #, "--experiment 'test'", '--overwrite False', + # '--setup-only True', '--rerun False', '--run-type ad', '--experiment test'] + description = "" cesmroot = path_to_ctsm_root() valid_neon_sites = glob.glob( os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*") @@ -57,11 +58,9 @@ def test_function(self): parsed_arguments = get_parser(sys.argv, description, valid_neon_sites) print(parsed_arguments) - self.assertEqual( - parsed_arguments[0], "ABBY", "arguments not processed as expected" - ) + self.assertEqual(parsed_arguments[0], "ABBY", "arguments not processed as expected") # TODO: Still need to figure out correct formatting to get argument recognized properly! - # TODO: Also it might be useful to add in a number of fake arguments to check that they all work... + # TODO: It might be useful to add a number of arguments to check that they all work... if __name__ == "__main__": diff --git a/python/ctsm/test/test_unit_NeonSite.py b/python/ctsm/test/test_unit_neon_site.py similarity index 54% rename from python/ctsm/test/test_unit_NeonSite.py rename to python/ctsm/test/test_unit_neon_site.py index 1520a58426..a75326e185 100755 --- a/python/ctsm/test/test_unit_NeonSite.py +++ b/python/ctsm/test/test_unit_neon_site.py @@ -3,7 +3,7 @@ Unit tests for NeonSite You can run this by: - python -m unittest test_unit_NeonSite.py + python -m unittest test_unit_neon_site.py """ import unittest @@ -45,89 +45,97 @@ def test_build_base_case(self): """ Test that NeonSite class' build_base_case is working properly... """ - #neonsite = NeonSite(ADD SOME PARAMETERS) - #neonsite.build_base_case(ARGUMENTS) # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? - #CHECK IF ACTS AS EXPECTED - #continue + # neonsite = NeonSite(ADD SOME PARAMETERS) + # neonsite.build_base_case(ARGUMENTS) + # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? + # CHECK IF ACTS AS EXPECTED + # continue def test_get_batch_query(self): """ Test that NeonSite class' get_batch_query is working properly... """ - #neonsite = NeonSite(ADD SOME PARAMETERS) - #neonsite.get_batch_query(ARGUMENTS) - #CHECK IF ACTS AS EXPECTED - #continue + # neonsite = NeonSite(ADD SOME PARAMETERS) + # neonsite.get_batch_query(ARGUMENTS) + # CHECK IF ACTS AS EXPECTED + # continue # ALSO DOESN'T SEEM THE MOST REASONABLE TO TEST def test_run_case(self): """ Test that NeonSite class' run_case is working properly... """ - #neonsite = NeonSite(ADD SOME PARAMETERS) - #neonsite.run_case(ARGUMENTS) # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? - #CHECK IF ACTS AS EXPECTED - #continue + # neonsite = NeonSite(ADD SOME PARAMETERS) + # neonsite.run_case(ARGUMENTS) + # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? + # CHECK IF ACTS AS EXPECTED + # continue def test_modify_user_nl_transient(self): """ - Test that NeonSite class' modify_user_nl is correctly adding lines to namelist for transient cases + Test that modify_user_nl is correctly adding lines to namelist for transient cases """ # NeonSite parameters: - name = 'ABBY' + name = "ABBY" start_year = 2020 end_year = 2021 start_month = 1 end_month = 12 - #finidat = None - finidat = 'dummy_finidat' + # finidat = None + finidat = "dummy_finidat" # modify_user_nl parameters: case_root = self._tempdir - run_type = 'transient' - rundir = '' + run_type = "transient" + rundir = "" - # update namelist - neonsite = NeonSite(name, start_year, end_year, start_month, end_month, finidat) - modified_neonsite = neonsite.modify_user_nl(case_root, run_type, rundir) + # create NeonSite object and update namelist + NeonSite(name, start_year, end_year, start_month, end_month, finidat).modify_user_nl( + case_root, run_type, rundir + ) # gather file contents for test - new_nl_file = open(glob.glob(case_root+'/*')[0], "r") + new_nl_file = open(glob.glob(case_root + "/*")[0], "r") lines_read = new_nl_file.readlines()[0] new_nl_file.close() - + # assertion - self.assertEqual(lines_read, "finidat = '/inputdata/lnd/ctsm/initdata/dummy_finidat'\n", 'transient case has unexpected nl') + self.assertEqual( + lines_read, + "finidat = '/inputdata/lnd/ctsm/initdata/dummy_finidat'\n", + "transient case has unexpected nl", + ) def test_modify_user_nl_ad(self): """ - Test that NeonSite class' modify_user_nl is correctly adding lines to namelist for ad cases + Test that modify_user_nl is correctly adding lines to namelist for ad cases """ # NeonSite parameters: - name = 'ABBY' + name = "ABBY" start_year = 2020 end_year = 2021 start_month = 1 end_month = 12 - #finidat = None - finidat = 'dummy_finidat' + # finidat = None + finidat = "dummy_finidat" # modify_user_nl parameters: case_root = self._tempdir - run_type = 'ad' - rundir = '' + run_type = "ad" + rundir = "" - # update namelist - neonsite = NeonSite(name, start_year, end_year, start_month, end_month, finidat) - modified_neonsite = neonsite.modify_user_nl(case_root, run_type, rundir) + # create NeonSite object and update namelist + NeonSite(name, start_year, end_year, start_month, end_month, finidat).modify_user_nl( + case_root, run_type, rundir + ) # gather file contents for test - new_nl_file = open(glob.glob(case_root+'/*')[0], "r") + new_nl_file = open(glob.glob(case_root + "/*")[0], "r") lines_read = new_nl_file.readlines()[1] new_nl_file.close() # assertion - self.assertEqual(lines_read, "hist_mfilt = 20\n", 'ad case has unexpected nl') + self.assertEqual(lines_read, "hist_mfilt = 20\n", "ad case has unexpected nl") if __name__ == "__main__": diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon index ad930f50e3..2a3091f6a6 100755 --- a/tools/site_and_regional/run_neon +++ b/tools/site_and_regional/run_neon @@ -41,6 +41,7 @@ _CTSM_PYTHON = os.path.join( ) sys.path.insert(1, _CTSM_PYTHON) +#pylint: disable=import-error, wrong-import-position from ctsm.site_and_regional.run_neon import main if __name__ == "__main__": From 1c12818024c2d9dfe79481364775e87984646bf0 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 12 Jan 2024 13:35:49 -0700 Subject: [PATCH 5/7] mostly testing updates --- python/ctsm/site_and_regional/arg_parse.py | 6 ++++- python/ctsm/site_and_regional/neon_site.py | 14 +++++----- python/ctsm/site_and_regional/run_neon.py | 4 +-- python/ctsm/test/test_unit_arg_parse.py | 12 ++++----- python/ctsm/test/test_unit_neon_site.py | 30 ---------------------- tools/site_and_regional/run_neon | 2 +- 6 files changed, 20 insertions(+), 48 deletions(-) diff --git a/python/ctsm/site_and_regional/arg_parse.py b/python/ctsm/site_and_regional/arg_parse.py index f45fef041c..99f184dd62 100644 --- a/python/ctsm/site_and_regional/arg_parse.py +++ b/python/ctsm/site_and_regional/arg_parse.py @@ -11,12 +11,13 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) -#pylint: disable=wrong-import-position, import-error, unused-import +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.utils import parse_isoduration from CIME.utils import parse_args_and_handle_standard_logging_options from CIME.utils import setup_standard_logging_options + def get_parser(args, description, valid_neon_sites): """ Get parser object for this script. @@ -209,9 +210,12 @@ def get_parser(args, description, valid_neon_sites): run_length = args.run_length run_length = parse_isoduration(run_length) + base_case_root = None if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) + if not os.path.exists(base_case_root): + raise ValueError("Base case root does not exist: {}".format(base_case_root)) # Reduce output level for this script unless --debug or # --verbose is provided on the command line diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 8881ef5867..31ae78f5ad 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -15,7 +15,7 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) -#pylint: disable=wrong-import-position, import-error, unused-import +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root @@ -26,6 +26,7 @@ logger = logging.getLogger(__name__) +# pylint: disable=too-many-instance-attributes class NeonSite: """ A class for encapsulating neon sites. @@ -40,11 +41,6 @@ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): self.cesmroot = path_to_ctsm_root() self.finidat = finidat - def __str__(self): - return ( - str(self.__class__) + "\n" + "\n".join((str(item) + " = " for item in (self.__dict__))) - ) - def build_base_case( self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False ): @@ -66,7 +62,9 @@ def build_base_case( Flag to overwrite the case if exists """ print("---- building a base case -------") + # pylint: disable=attribute-defined-outside-init self.base_case_root = output_root + # pylint: enable=attribute-defined-outside-init user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] if not output_root: output_root = os.getcwd() @@ -135,6 +133,7 @@ def build_base_case( # update case_path to be the full path to the base case return case_path + # pylint: disable=no-self-use def get_batch_query(self, case): """ Function for querying the batch queue query command for a case, depending on the @@ -149,6 +148,7 @@ def get_batch_query(self, case): return "none" return case.get_value("batch_query") + # pylint: disable=too-many-statements def run_case( self, base_case_root, @@ -218,12 +218,14 @@ def run_case( # For existing case check that the compset name is correct existingcompname = case.get_value("COMPSET") match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + # pylint: disable=undefined-variable if re.search("^HIST", compset, flags=re.IGNORECASE) is None: expect( match is None, """Existing base case is a historical type and should not be --rerun with the --overwrite option""", ) + # pylint: enable=undefined-variable else: expect( match is not None, diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 3802133a64..31c5bbafb4 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -67,10 +67,8 @@ from ctsm.site_and_regional.arg_parse import get_parser from ctsm.site_and_regional.neon_site import NeonSite -# pylint: disable=import-error +# pylint: disable=import-error, wildcard-import, wrong-import-order from standard_script_setup import * -# the above import is need to set CIMEROOT environment variable -# TODO: figure out what specifically needs to be imported (not '*') logger = logging.getLogger(__name__) diff --git a/python/ctsm/test/test_unit_arg_parse.py b/python/ctsm/test/test_unit_arg_parse.py index 1da6b63f77..71254eee60 100755 --- a/python/ctsm/test/test_unit_arg_parse.py +++ b/python/ctsm/test/test_unit_arg_parse.py @@ -46,9 +46,7 @@ def test_function(self): """ Test that arg_parse is working properly... """ - sys.argv = ["--neon-sites ['ABBY']"] - # arguments= ["--neon-sites", "ABBY"] #, "--experiment 'test'", '--overwrite False', - # '--setup-only True', '--rerun False', '--run-type ad', '--experiment test'] + sys.argv = ["arg_parse", "--neon-sites", "ABBY", "--experiment", "test", "--run-type", "ad"] description = "" cesmroot = path_to_ctsm_root() valid_neon_sites = glob.glob( @@ -57,10 +55,10 @@ def test_function(self): valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) parsed_arguments = get_parser(sys.argv, description, valid_neon_sites) - print(parsed_arguments) - self.assertEqual(parsed_arguments[0], "ABBY", "arguments not processed as expected") - # TODO: Still need to figure out correct formatting to get argument recognized properly! - # TODO: It might be useful to add a number of arguments to check that they all work... + self.assertEqual(parsed_arguments[0][0], "ABBY", "arguments not processed as expected") + self.assertEqual(parsed_arguments[3], "test", "arguments not processed as expected") + self.assertEqual(parsed_arguments[4], False, "arguments not processed as expected") + self.assertEqual(parsed_arguments[2], "ad", "arguments not processed as expected") if __name__ == "__main__": diff --git a/python/ctsm/test/test_unit_neon_site.py b/python/ctsm/test/test_unit_neon_site.py index a75326e185..4828718272 100755 --- a/python/ctsm/test/test_unit_neon_site.py +++ b/python/ctsm/test/test_unit_neon_site.py @@ -41,36 +41,6 @@ def tearDown(self): """ shutil.rmtree(self._tempdir, ignore_errors=True) - def test_build_base_case(self): - """ - Test that NeonSite class' build_base_case is working properly... - """ - # neonsite = NeonSite(ADD SOME PARAMETERS) - # neonsite.build_base_case(ARGUMENTS) - # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? - # CHECK IF ACTS AS EXPECTED - # continue - - def test_get_batch_query(self): - """ - Test that NeonSite class' get_batch_query is working properly... - """ - # neonsite = NeonSite(ADD SOME PARAMETERS) - # neonsite.get_batch_query(ARGUMENTS) - # CHECK IF ACTS AS EXPECTED - # continue - # ALSO DOESN'T SEEM THE MOST REASONABLE TO TEST - - def test_run_case(self): - """ - Test that NeonSite class' run_case is working properly... - """ - # neonsite = NeonSite(ADD SOME PARAMETERS) - # neonsite.run_case(ARGUMENTS) - # NOT SURE WE ACTUALLY WANT TO DO THIS DUE TO TIME CONSTRAINTS? - # CHECK IF ACTS AS EXPECTED - # continue - def test_modify_user_nl_transient(self): """ Test that modify_user_nl is correctly adding lines to namelist for transient cases diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon index 2a3091f6a6..ffc3be2af7 100755 --- a/tools/site_and_regional/run_neon +++ b/tools/site_and_regional/run_neon @@ -41,7 +41,7 @@ _CTSM_PYTHON = os.path.join( ) sys.path.insert(1, _CTSM_PYTHON) -#pylint: disable=import-error, wrong-import-position +# pylint: disable=import-error, wrong-import-position from ctsm.site_and_regional.run_neon import main if __name__ == "__main__": From 2d2df94cdc536ca80a925c3dde517516bea9d7c0 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 16 Jan 2024 12:20:31 -0700 Subject: [PATCH 6/7] rename arg_parse --- .../{arg_parse.py => neon_arg_parse.py} | 0 python/ctsm/site_and_regional/run_neon.py | 2 +- ...it_arg_parse.py => test_unit_neon_arg_parse.py} | 14 +++++++------- 3 files changed, 8 insertions(+), 8 deletions(-) rename python/ctsm/site_and_regional/{arg_parse.py => neon_arg_parse.py} (100%) rename python/ctsm/test/{test_unit_arg_parse.py => test_unit_neon_arg_parse.py} (80%) diff --git a/python/ctsm/site_and_regional/arg_parse.py b/python/ctsm/site_and_regional/neon_arg_parse.py similarity index 100% rename from python/ctsm/site_and_regional/arg_parse.py rename to python/ctsm/site_and_regional/neon_arg_parse.py diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 31c5bbafb4..72bf3fdfb4 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -64,7 +64,7 @@ # pylint: disable=wrong-import-position from ctsm.path_utils import path_to_ctsm_root from ctsm.download_utils import download_file -from ctsm.site_and_regional.arg_parse import get_parser +from ctsm.site_and_regional.neon_arg_parse import get_parser from ctsm.site_and_regional.neon_site import NeonSite # pylint: disable=import-error, wildcard-import, wrong-import-order diff --git a/python/ctsm/test/test_unit_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py similarity index 80% rename from python/ctsm/test/test_unit_arg_parse.py rename to python/ctsm/test/test_unit_neon_arg_parse.py index 71254eee60..863c7e3e5a 100755 --- a/python/ctsm/test/test_unit_arg_parse.py +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """ -Unit tests for arg_parse +Unit tests for neon_arg_parse You can run this by: - python -m unittest test_unit_arg_parse.py + python -m unittest test_unit_neon_arg_parse.py """ import unittest @@ -19,15 +19,15 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.site_and_regional.arg_parse import get_parser +from ctsm.site_and_regional.neon_arg_parse import get_parser from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name -class Testarg_parse(unittest.TestCase): +class Test_neon_arg_parse(unittest.TestCase): """ - Basic class for testing arg_parse.py. + Basic class for testing neon_arg_parse.py. """ def setUp(self): @@ -44,9 +44,9 @@ def tearDown(self): def test_function(self): """ - Test that arg_parse is working properly... + Test that neon_arg_parse is properly reading arguments... """ - sys.argv = ["arg_parse", "--neon-sites", "ABBY", "--experiment", "test", "--run-type", "ad"] + sys.argv = ["neon_arg_parse", "--neon-sites", "ABBY", "--experiment", "test", "--run-type", "ad"] description = "" cesmroot = path_to_ctsm_root() valid_neon_sites = glob.glob( From a59c6e12d5d7cb2bd438bda5f536a9e668e022e4 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 16 Jan 2024 12:32:19 -0700 Subject: [PATCH 7/7] black arg_parse unit test --- python/ctsm/test/test_unit_neon_arg_parse.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/ctsm/test/test_unit_neon_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py index 863c7e3e5a..7bae337709 100755 --- a/python/ctsm/test/test_unit_neon_arg_parse.py +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -46,7 +46,15 @@ def test_function(self): """ Test that neon_arg_parse is properly reading arguments... """ - sys.argv = ["neon_arg_parse", "--neon-sites", "ABBY", "--experiment", "test", "--run-type", "ad"] + sys.argv = [ + "neon_arg_parse", + "--neon-sites", + "ABBY", + "--experiment", + "test", + "--run-type", + "ad", + ] description = "" cesmroot = path_to_ctsm_root() valid_neon_sites = glob.glob(