From 2cdb380febb274478e84cd90945aee93f29fa2e6 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 13 Dec 2021 01:29:05 -0700 Subject: [PATCH 01/16] Run python directory through black 21.11b1 --- python/ctsm/add_cime_to_path.py | 1 + python/ctsm/ctsm_logging.py | 19 +- python/ctsm/joblauncher/job_launcher_base.py | 38 +- .../ctsm/joblauncher/job_launcher_factory.py | 40 +- python/ctsm/joblauncher/job_launcher_fake.py | 19 +- .../ctsm/joblauncher/job_launcher_no_batch.py | 23 +- python/ctsm/joblauncher/job_launcher_qsub.py | 60 +- python/ctsm/lilac_build_ctsm.py | 856 +++++++++++------- python/ctsm/lilac_download_input_data.py | 57 +- python/ctsm/lilac_make_runtime_inputs.py | 294 +++--- python/ctsm/machine.py | 94 +- python/ctsm/machine_defaults.py | 84 +- python/ctsm/machine_utils.py | 10 +- python/ctsm/os_utils.py | 25 +- python/ctsm/path_utils.py | 42 +- python/ctsm/run_ctsm_py_tests.py | 49 +- python/ctsm/run_sys_tests.py | 742 +++++++++------ .../test_unit_job_launcher_no_batch.py | 30 +- python/ctsm/test/test_sys_lilac_build_ctsm.py | 76 +- .../ctsm/test/test_unit_lilac_build_ctsm.py | 239 +++-- .../test_unit_lilac_make_runtime_inputs.py | 35 +- python/ctsm/test/test_unit_machine.py | 235 ++--- python/ctsm/test/test_unit_path_utils.py | 51 +- python/ctsm/test/test_unit_run_sys_tests.py | 266 +++--- python/ctsm/test/test_unit_utils.py | 13 +- python/ctsm/unit_testing.py | 1 + python/ctsm/utils.py | 6 +- python/six.py | 201 ++-- python/six_additions.py | 1 + 29 files changed, 2175 insertions(+), 1432 deletions(-) diff --git a/python/ctsm/add_cime_to_path.py b/python/ctsm/add_cime_to_path.py index a18f19abfd..dba4bf1477 100644 --- a/python/ctsm/add_cime_to_path.py +++ b/python/ctsm/add_cime_to_path.py @@ -22,4 +22,5 @@ """ from ctsm.path_utils import add_cime_lib_to_path + _ = add_cime_lib_to_path() diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index 7d63b9b463..50dc479a4f 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -31,6 +31,7 @@ logger = logging.getLogger(__name__) + def setup_logging_pre_config(): """Setup logging for a script / application @@ -41,12 +42,14 @@ def setup_logging_pre_config(): """ setup_logging(level=logging.WARNING) + def setup_logging_for_tests(enable_critical=False): """Setup logging as appropriate for unit tests""" setup_logging(level=logging.CRITICAL) if not enable_critical: logging.disable(logging.CRITICAL) + def setup_logging(level=logging.WARNING): """Setup logging for a script / application @@ -54,18 +57,24 @@ def setup_logging(level=logging.WARNING): do NOT intend to allow the user to control logging preferences via command-line arguments, so that all of the final logging options are set here. """ - logging.basicConfig(format='%(levelname)s: %(message)s', level=level) + logging.basicConfig(format="%(levelname)s: %(message)s", level=level) + def add_logging_args(parser): """Add common logging-related options to the argument parser""" logging_level = parser.add_mutually_exclusive_group() - logging_level.add_argument('-v', '--verbose', action='store_true', - help='Output extra logging info') + logging_level.add_argument( + "-v", "--verbose", action="store_true", help="Output extra logging info" + ) + + logging_level.add_argument( + "--debug", + action="store_true", + help="Output even more logging info for debugging", + ) - logging_level.add_argument('--debug', action='store_true', - help='Output even more logging info for debugging') def process_logging_args(args): """Configure logging based on the logging-related args added by add_logging_args""" diff --git a/python/ctsm/joblauncher/job_launcher_base.py b/python/ctsm/joblauncher/job_launcher_base.py index 026a4e736d..86075aa27e 100644 --- a/python/ctsm/joblauncher/job_launcher_base.py +++ b/python/ctsm/joblauncher/job_launcher_base.py @@ -11,11 +11,18 @@ logger = logging.getLogger(__name__) + class JobLauncherBase(object): """Base class for job launchers. Not meant to be instantiated directly""" - def __init__(self, queue=None, walltime=None, account=None, - required_args=None, extra_args=None): + def __init__( + self, + queue=None, + walltime=None, + account=None, + required_args=None, + extra_args=None, + ): """Initialize a job launcher object. Note that some of these arguments (e.g., queue and walltime) aren't needed by some @@ -84,7 +91,9 @@ def run_command(self, command, stdout_path, stderr_path, dry_run=False): If dry_run is True, then just print the command to be run without actually running it. """ - logger.info("%s", self.run_command_logger_message(command, stdout_path, stderr_path)) + logger.info( + "%s", self.run_command_logger_message(command, stdout_path, stderr_path) + ) if not dry_run: self.run_command_impl(command, stdout_path, stderr_path) @@ -109,13 +118,16 @@ def run_command_logger_message(self, command, stdout_path, stderr_path): raise NotImplementedError def __repr__(self): - return (type(self).__name__ + - "(queue='{queue}', " - "walltime='{walltime}', " - "account='{account}', " - "required_args='{required_args}', " - "extra_args='{extra_args}')".format(queue=self._queue, - walltime=self._walltime, - account=self._account, - required_args=self._required_args, - extra_args=self._extra_args)) + return ( + type(self).__name__ + "(queue='{queue}', " + "walltime='{walltime}', " + "account='{account}', " + "required_args='{required_args}', " + "extra_args='{extra_args}')".format( + queue=self._queue, + walltime=self._walltime, + account=self._account, + required_args=self._required_args, + extra_args=self._extra_args, + ) + ) diff --git a/python/ctsm/joblauncher/job_launcher_factory.py b/python/ctsm/joblauncher/job_launcher_factory.py index 932771f666..9a615769fd 100644 --- a/python/ctsm/joblauncher/job_launcher_factory.py +++ b/python/ctsm/joblauncher/job_launcher_factory.py @@ -15,11 +15,17 @@ JOB_LAUNCHER_QSUB = "qsub" JOB_LAUNCHER_FAKE = "fake" -def create_job_launcher(job_launcher_type, - account=None, queue=None, walltime=None, - nice_level=None, - required_args=None, extra_args=None, - allow_missing_entries=False): + +def create_job_launcher( + job_launcher_type, + account=None, + queue=None, + walltime=None, + nice_level=None, + required_args=None, + extra_args=None, + allow_missing_entries=False, +): """ Creates and returns a job launcher object of the specified type @@ -52,18 +58,24 @@ def create_job_launcher(job_launcher_type, if job_launcher_type == JOB_LAUNCHER_QSUB: if not allow_missing_entries: - _assert_not_none(queue, 'queue', job_launcher_type) - _assert_not_none(walltime, 'walltime', job_launcher_type) - return JobLauncherQsub(queue=queue, - walltime=walltime, - account=account, - required_args=required_args, - extra_args=extra_args) + _assert_not_none(queue, "queue", job_launcher_type) + _assert_not_none(walltime, "walltime", job_launcher_type) + return JobLauncherQsub( + queue=queue, + walltime=walltime, + account=account, + required_args=required_args, + extra_args=extra_args, + ) raise RuntimeError("Unexpected job launcher type: {}".format(job_launcher_type)) + def _assert_not_none(arg, arg_name, job_launcher_type): """Raises an exception if the given argument has value None""" if arg is None: - raise TypeError("{} cannot be None for job launcher of type {}".format( - arg_name, job_launcher_type)) + raise TypeError( + "{} cannot be None for job launcher of type {}".format( + arg_name, job_launcher_type + ) + ) diff --git a/python/ctsm/joblauncher/job_launcher_fake.py b/python/ctsm/joblauncher/job_launcher_fake.py index 0546016879..8c52bf7688 100644 --- a/python/ctsm/joblauncher/job_launcher_fake.py +++ b/python/ctsm/joblauncher/job_launcher_fake.py @@ -6,7 +6,8 @@ # cmd (str): space-delimited string giving this command # out (str): path to stdout # err (str): path to stderr -Command = namedtuple('Command', ['cmd', 'out', 'err']) +Command = namedtuple("Command", ["cmd", "out", "err"]) + class JobLauncherFake(JobLauncherBase): """A fake JobLauncher that just records the commands it is told to run""" @@ -16,16 +17,16 @@ def __init__(self): self._commands = [] def run_command_impl(self, command, stdout_path, stderr_path): - self._commands.append(Command(cmd=' '.join(command), - out=stdout_path, - err=stderr_path)) + self._commands.append( + Command(cmd=" ".join(command), out=stdout_path, err=stderr_path) + ) def run_command_logger_message(self, command, stdout_path, stderr_path): - message = 'Appending: <{}> ' \ - 'with stdout = {} ' \ - 'and stderr = {}'.format(' '.join(command), - stdout_path, - stderr_path) + message = ( + "Appending: <{}> " + "with stdout = {} " + "and stderr = {}".format(" ".join(command), stdout_path, stderr_path) + ) return message def get_commands(self): diff --git a/python/ctsm/joblauncher/job_launcher_no_batch.py b/python/ctsm/joblauncher/job_launcher_no_batch.py index 3ca34afc86..6b307dbc42 100644 --- a/python/ctsm/joblauncher/job_launcher_no_batch.py +++ b/python/ctsm/joblauncher/job_launcher_no_batch.py @@ -5,6 +5,7 @@ import os from ctsm.joblauncher.job_launcher_base import JobLauncherBase + class JobLauncherNoBatch(JobLauncherBase): """Job launcher for systems where we can run a big job directly on the login node @@ -42,22 +43,22 @@ def _preexec(self): os.nice(self._nice_level) def run_command_impl(self, command, stdout_path, stderr_path): - with open(stdout_path, 'w') as outfile, \ - open(stderr_path, 'w') as errfile: + with open(stdout_path, "w") as outfile, open(stderr_path, "w") as errfile: # Note that preexec_fn is POSIX-only; also, it may be unsafe in the presence # of threads (hence the need for disabling the pylint warning) # pylint: disable=subprocess-popen-preexec-fn - self._process = subprocess.Popen(command, - stdout=outfile, - stderr=errfile, - preexec_fn=self._preexec) + self._process = subprocess.Popen( + command, stdout=outfile, stderr=errfile, preexec_fn=self._preexec + ) def run_command_logger_message(self, command, stdout_path, stderr_path): - message = 'Running: <{command_str}> ' \ - 'with stdout = {outfile} ' \ - 'and stderr = {errfile}'.format(command_str=' '.join(command), - outfile=stdout_path, - errfile=stderr_path) + message = ( + "Running: <{command_str}> " + "with stdout = {outfile} " + "and stderr = {errfile}".format( + command_str=" ".join(command), outfile=stdout_path, errfile=stderr_path + ) + ) return message def wait_for_last_process_to_complete(self): diff --git a/python/ctsm/joblauncher/job_launcher_qsub.py b/python/ctsm/joblauncher/job_launcher_qsub.py index bafd3d00f5..779911400d 100644 --- a/python/ctsm/joblauncher/job_launcher_qsub.py +++ b/python/ctsm/joblauncher/job_launcher_qsub.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + class JobLauncherQsub(JobLauncherBase): """Job launcher for systems where we run big jobs via qsub""" @@ -14,42 +15,51 @@ def __init__(self, queue, walltime, account, required_args, extra_args): """ account can be None or the empty string on a machine that doesn't require an account """ - JobLauncherBase.__init__(self, - queue=queue, - walltime=walltime, - account=account, - required_args=required_args, - extra_args=extra_args) + JobLauncherBase.__init__( + self, + queue=queue, + walltime=walltime, + account=account, + required_args=required_args, + extra_args=extra_args, + ) def run_command_impl(self, command, stdout_path, stderr_path): - qsub_process = subprocess.Popen(self._qsub_command(stdout_path, stderr_path), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - (out, err) = qsub_process.communicate(' '.join(command)) + qsub_process = subprocess.Popen( + self._qsub_command(stdout_path, stderr_path), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + (out, err) = qsub_process.communicate(" ".join(command)) if err: - logger.info('qsub ERROR:\n%s', err) + logger.info("qsub ERROR:\n%s", err) if out: - logger.info('qsub OUTPUT:\n%s', out) + logger.info("qsub OUTPUT:\n%s", out) def run_command_logger_message(self, command, stdout_path, stderr_path): - message = 'Running: <{command_str}> ' \ - 'via <{qsub_str}>'.format( - command_str=' '.join(command), - qsub_str=' '.join(self._qsub_command(stdout_path, - stderr_path))) + message = "Running: <{command_str}> " "via <{qsub_str}>".format( + command_str=" ".join(command), + qsub_str=" ".join(self._qsub_command(stdout_path, stderr_path)), + ) return message def _qsub_command(self, stdout_path, stderr_path): """Returns the qsub command""" - qsub_cmd = ['qsub', - '-q', self._queue, - '-l', 'walltime={}'.format(self._walltime), - '-o', stdout_path, - '-e', stderr_path] + qsub_cmd = [ + "qsub", + "-q", + self._queue, + "-l", + "walltime={}".format(self._walltime), + "-o", + stdout_path, + "-e", + stderr_path, + ] if self._account: - qsub_cmd.extend(['-A', self._account]) + qsub_cmd.extend(["-A", self._account]) if self._required_args: qsub_cmd.extend(self._required_args.split()) if self._extra_args: diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index be1fbf7350..99e09d0614 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -6,7 +6,11 @@ import shutil import subprocess -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) from ctsm.os_utils import run_cmd_output_on_error, make_link from ctsm.path_utils import path_to_ctsm_root from ctsm.utils import abort, fill_template_file @@ -18,35 +22,34 @@ # ======================================================================== # this matches the machine name in config_machines_template.xml -_MACH_NAME = 'ctsm-build' +_MACH_NAME = "ctsm-build" # these are arbitrary, since we only use the case for its build, not any of the runtime # settings; they just need to be valid -_COMPSET = 'I2000Ctsm50NwpSpAsRs' -_RES = 'f10_f10_mg37' +_COMPSET = "I2000Ctsm50NwpSpAsRs" +_RES = "f10_f10_mg37" -_PATH_TO_TEMPLATES = os.path.join(path_to_ctsm_root(), - 'lilac', - 'bld_templates') +_PATH_TO_TEMPLATES = os.path.join(path_to_ctsm_root(), "lilac", "bld_templates") -_PATH_TO_MAKE_RUNTIME_INPUTS = os.path.join(path_to_ctsm_root(), - 'lilac', - 'make_runtime_inputs') +_PATH_TO_MAKE_RUNTIME_INPUTS = os.path.join( + path_to_ctsm_root(), "lilac", "make_runtime_inputs" +) -_PATH_TO_DOWNLOAD_INPUT_DATA = os.path.join(path_to_ctsm_root(), - 'lilac', - 'download_input_data') +_PATH_TO_DOWNLOAD_INPUT_DATA = os.path.join( + path_to_ctsm_root(), "lilac", "download_input_data" +) -_MACHINE_CONFIG_DIRNAME = 'machine_configuration' -_INPUTDATA_DIRNAME = 'inputdata' -_RUNTIME_INPUTS_DIRNAME = 'runtime_inputs' +_MACHINE_CONFIG_DIRNAME = "machine_configuration" +_INPUTDATA_DIRNAME = "inputdata" +_RUNTIME_INPUTS_DIRNAME = "runtime_inputs" -_GPTL_NANOTIMERS_CPPDEFS = '-DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY' # pylint: disable=line-too-long +_GPTL_NANOTIMERS_CPPDEFS = "-DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY" # pylint: disable=line-too-long # ======================================================================== # Public functions # ======================================================================== + def main(cime_path): """Main function called when build_ctsm is run from the command-line @@ -64,47 +67,52 @@ def main(cime_path): if args.rebuild: rebuild_ctsm(build_dir=build_dir) else: - build_ctsm(cime_path=cime_path, - build_dir=build_dir, - compiler=args.compiler, - no_build=args.no_build, - machine=args.machine, - os_type=args.os, - netcdf_path=args.netcdf_path, - esmf_mkfile_path=args.esmf_mkfile_path, - max_mpitasks_per_node=args.max_mpitasks_per_node, - gmake=args.gmake, - gmake_j=args.gmake_j, - pnetcdf_path=args.pnetcdf_path, - pio_filesystem_hints=args.pio_filesystem_hints, - gptl_nano_timers=args.gptl_nano_timers, - extra_fflags=args.extra_fflags, - extra_cflags=args.extra_cflags, - no_pnetcdf=args.no_pnetcdf, - build_debug=args.build_debug, - build_with_openmp=args.build_with_openmp, - inputdata_path=args.inputdata_path) - -def build_ctsm(cime_path, - build_dir, - compiler, - no_build=False, - machine=None, - os_type=None, - netcdf_path=None, - esmf_mkfile_path=None, - max_mpitasks_per_node=None, - gmake=None, - gmake_j=None, - pnetcdf_path=None, - pio_filesystem_hints=None, - gptl_nano_timers=False, - extra_fflags='', - extra_cflags='', - no_pnetcdf=False, - build_debug=False, - build_with_openmp=False, - inputdata_path=None): + build_ctsm( + cime_path=cime_path, + build_dir=build_dir, + compiler=args.compiler, + no_build=args.no_build, + machine=args.machine, + os_type=args.os, + netcdf_path=args.netcdf_path, + esmf_mkfile_path=args.esmf_mkfile_path, + max_mpitasks_per_node=args.max_mpitasks_per_node, + gmake=args.gmake, + gmake_j=args.gmake_j, + pnetcdf_path=args.pnetcdf_path, + pio_filesystem_hints=args.pio_filesystem_hints, + gptl_nano_timers=args.gptl_nano_timers, + extra_fflags=args.extra_fflags, + extra_cflags=args.extra_cflags, + no_pnetcdf=args.no_pnetcdf, + build_debug=args.build_debug, + build_with_openmp=args.build_with_openmp, + inputdata_path=args.inputdata_path, + ) + + +def build_ctsm( + cime_path, + build_dir, + compiler, + no_build=False, + machine=None, + os_type=None, + netcdf_path=None, + esmf_mkfile_path=None, + max_mpitasks_per_node=None, + gmake=None, + gmake_j=None, + pnetcdf_path=None, + pio_filesystem_hints=None, + gptl_nano_timers=False, + extra_fflags="", + extra_cflags="", + no_pnetcdf=False, + build_debug=False, + build_with_openmp=False, + inputdata_path=None, +): """Implementation of build_ctsm command Args: @@ -147,46 +155,55 @@ def build_ctsm(cime_path, existing_machine = machine is not None existing_inputdata = existing_machine or inputdata_path is not None - _create_build_dir(build_dir=build_dir, - existing_inputdata=existing_inputdata) + _create_build_dir(build_dir=build_dir, existing_inputdata=existing_inputdata) if machine is None: - assert os_type is not None, 'with machine absent, os_type must be given' - assert netcdf_path is not None, 'with machine absent, netcdf_path must be given' - assert esmf_mkfile_path is not None, 'with machine absent, esmf_mkfile_path must be given' - assert max_mpitasks_per_node is not None, ('with machine absent ' - 'max_mpitasks_per_node must be given') + assert os_type is not None, "with machine absent, os_type must be given" + assert netcdf_path is not None, "with machine absent, netcdf_path must be given" + assert ( + esmf_mkfile_path is not None + ), "with machine absent, esmf_mkfile_path must be given" + assert max_mpitasks_per_node is not None, ( + "with machine absent " "max_mpitasks_per_node must be given" + ) os_type = _check_and_transform_os(os_type) - _fill_out_machine_files(build_dir=build_dir, - os_type=os_type, - compiler=compiler, - netcdf_path=netcdf_path, - esmf_mkfile_path=esmf_mkfile_path, - max_mpitasks_per_node=max_mpitasks_per_node, - gmake=gmake, - gmake_j=gmake_j, - pnetcdf_path=pnetcdf_path, - pio_filesystem_hints=pio_filesystem_hints, - gptl_nano_timers=gptl_nano_timers, - extra_fflags=extra_fflags, - extra_cflags=extra_cflags) - - _create_case(cime_path=cime_path, - build_dir=build_dir, - compiler=compiler, - machine=machine, - build_debug=build_debug, - build_with_openmp=build_with_openmp, - inputdata_path=inputdata_path) + _fill_out_machine_files( + build_dir=build_dir, + os_type=os_type, + compiler=compiler, + netcdf_path=netcdf_path, + esmf_mkfile_path=esmf_mkfile_path, + max_mpitasks_per_node=max_mpitasks_per_node, + gmake=gmake, + gmake_j=gmake_j, + pnetcdf_path=pnetcdf_path, + pio_filesystem_hints=pio_filesystem_hints, + gptl_nano_timers=gptl_nano_timers, + extra_fflags=extra_fflags, + extra_cflags=extra_cflags, + ) + + _create_case( + cime_path=cime_path, + build_dir=build_dir, + compiler=compiler, + machine=machine, + build_debug=build_debug, + build_with_openmp=build_with_openmp, + inputdata_path=inputdata_path, + ) _stage_runtime_inputs(build_dir=build_dir, no_pnetcdf=no_pnetcdf) - print('Initial setup complete; it is now safe to work with the runtime inputs in\n' - '{}\n'.format(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME))) + print( + "Initial setup complete; it is now safe to work with the runtime inputs in\n" + "{}\n".format(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME)) + ) if not no_build: _build_case(build_dir=build_dir) + def rebuild_ctsm(build_dir): """Re-run the build in an existing directory @@ -194,30 +211,37 @@ def rebuild_ctsm(build_dir): build_dir (str): path to build directory """ if not os.path.exists(build_dir): - abort('When running with --rebuild, the build directory must already exist\n' - '(<{}> does not exist)'.format(build_dir)) + abort( + "When running with --rebuild, the build directory must already exist\n" + "(<{}> does not exist)".format(build_dir) + ) case_dir = _get_case_dir(build_dir) if not os.path.exists(case_dir): - abort('It appears there was a problem setting up the initial build in\n' - '<{}>\n' - 'You should start over with a fresh build directory.'.format(build_dir)) + abort( + "It appears there was a problem setting up the initial build in\n" + "<{}>\n" + "You should start over with a fresh build directory.".format(build_dir) + ) try: subprocess.check_call( - [os.path.join(case_dir, 'case.build'), - '--clean-depends', - 'lnd'], - cwd=case_dir) + [os.path.join(case_dir, "case.build"), "--clean-depends", "lnd"], + cwd=case_dir, + ) except subprocess.CalledProcessError: - abort('ERROR resetting build for CTSM in order to rebuild - see above for details') + abort( + "ERROR resetting build for CTSM in order to rebuild - see above for details" + ) _build_case(build_dir) + # ======================================================================== # Private functions # ======================================================================== + def _commandline_args(args_to_parse=None): """Parse and return command-line arguments @@ -256,176 +280,254 @@ def _commandline_args(args_to_parse=None): """ parser = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter) + description=description, formatter_class=argparse.RawTextHelpFormatter + ) - parser.add_argument('build_dir', - help='Path to build directory\n' - 'If --rebuild is given, this should be the path to an existing build,\n' - 'otherwise this directory must not already exist.') + parser.add_argument( + "build_dir", + help="Path to build directory\n" + "If --rebuild is given, this should be the path to an existing build,\n" + "otherwise this directory must not already exist.", + ) main_opts = parser.add_mutually_exclusive_group() - main_opts.add_argument('--machine', - help='Name of machine; this must be a machine that has been ported to cime\n' - '(http://esmci.github.io/cime/versions/master/html/users_guide/porting-cime.html)\n' - 'If given, then none of the machine-definition optional arguments should be given.\n') - - main_opts.add_argument('--rebuild', action='store_true', - help='Rebuild in an existing build directory\n' - 'If given, none of the machine-definition or build-related optional arguments\n' - 'should be given.\n') + main_opts.add_argument( + "--machine", + help="Name of machine; this must be a machine that has been ported to cime\n" + "(http://esmci.github.io/cime/versions/master/html/users_guide/porting-cime.html)\n" + "If given, then none of the machine-definition optional arguments should be given.\n", + ) + + main_opts.add_argument( + "--rebuild", + action="store_true", + help="Rebuild in an existing build directory\n" + "If given, none of the machine-definition or build-related optional arguments\n" + "should be given.\n", + ) non_rebuild_required = parser.add_argument_group( - title='required arguments when not rebuilding', - description='These arguments are required if --rebuild is not given; ' - 'they are not allowed with --rebuild:') + title="required arguments when not rebuilding", + description="These arguments are required if --rebuild is not given; " + "they are not allowed with --rebuild:", + ) non_rebuild_required_list = [] # For now, only support the compilers that we regularly test with, even though cime # supports many other options - non_rebuild_required.add_argument('--compiler', type=str.lower, - choices=['gnu', 'intel', 'nag', 'pgi'], - help='Compiler type') - non_rebuild_required_list.append('compiler') + non_rebuild_required.add_argument( + "--compiler", + type=str.lower, + choices=["gnu", "intel", "nag", "pgi"], + help="Compiler type", + ) + non_rebuild_required_list.append("compiler") non_rebuild_optional = parser.add_argument_group( - title='optional arguments when not rebuilding', - description='These arguments are optional if --rebuild is not given; ' - 'they are not allowed with --rebuild:') + title="optional arguments when not rebuilding", + description="These arguments are optional if --rebuild is not given; " + "they are not allowed with --rebuild:", + ) non_rebuild_optional_list = [] - non_rebuild_optional.add_argument('--no-pnetcdf', action='store_true', - help='Use NetCDF instead of PNetCDF for CTSM I/O.\n' - 'On a user-defined machine, you must either set this flag\n' - 'or set --pnetcdf-path. On a cime-ported machine,\n' - 'this flag must be set if PNetCDF is not available\n' - 'for this machine/compiler.') - non_rebuild_optional_list.append('no-pnetcdf') - - non_rebuild_optional.add_argument('--build-debug', action='store_true', - help='Build with flags for debugging rather than production runs') - non_rebuild_optional_list.append('build-debug') - - non_rebuild_optional.add_argument('--build-with-openmp', action='store_true', - help='By default, CTSM is built WITHOUT support for OpenMP threading;\n' - 'if this flag is set, then CTSM is built WITH this support.\n' - 'This is important for performance if you will be running with\n' - 'OpenMP threading-based parallelization, or hybrid MPI/OpenMP.') - non_rebuild_optional_list.append('build-with-openmp') - - non_rebuild_optional.add_argument('--inputdata-path', - help='Path to directory containing CTSM\'s NetCDF inputs.\n' - 'For a machine that has been ported to cime, the default is to\n' - 'use this machine\'s standard inputdata location; this argument\n' - 'can be used to override this default.\n' - 'For a user-defined machine, the default is to create an inputdata\n' - 'directory in the build directory; again, this argument can be\n' - 'used to override this default.') - non_rebuild_optional_list.append('inputdata-path') - - non_rebuild_optional.add_argument('--no-build', action='store_true', - help='Do the pre-build setup, but do not actually build CTSM\n' - '(This is useful for testing, or for expert use.)') - non_rebuild_optional_list.append('no-build') + non_rebuild_optional.add_argument( + "--no-pnetcdf", + action="store_true", + help="Use NetCDF instead of PNetCDF for CTSM I/O.\n" + "On a user-defined machine, you must either set this flag\n" + "or set --pnetcdf-path. On a cime-ported machine,\n" + "this flag must be set if PNetCDF is not available\n" + "for this machine/compiler.", + ) + non_rebuild_optional_list.append("no-pnetcdf") + + non_rebuild_optional.add_argument( + "--build-debug", + action="store_true", + help="Build with flags for debugging rather than production runs", + ) + non_rebuild_optional_list.append("build-debug") + + non_rebuild_optional.add_argument( + "--build-with-openmp", + action="store_true", + help="By default, CTSM is built WITHOUT support for OpenMP threading;\n" + "if this flag is set, then CTSM is built WITH this support.\n" + "This is important for performance if you will be running with\n" + "OpenMP threading-based parallelization, or hybrid MPI/OpenMP.", + ) + non_rebuild_optional_list.append("build-with-openmp") + + non_rebuild_optional.add_argument( + "--inputdata-path", + help="Path to directory containing CTSM's NetCDF inputs.\n" + "For a machine that has been ported to cime, the default is to\n" + "use this machine's standard inputdata location; this argument\n" + "can be used to override this default.\n" + "For a user-defined machine, the default is to create an inputdata\n" + "directory in the build directory; again, this argument can be\n" + "used to override this default.", + ) + non_rebuild_optional_list.append("inputdata-path") + + non_rebuild_optional.add_argument( + "--no-build", + action="store_true", + help="Do the pre-build setup, but do not actually build CTSM\n" + "(This is useful for testing, or for expert use.)", + ) + non_rebuild_optional_list.append("no-build") new_machine_required = parser.add_argument_group( - title='required arguments for a user-defined machine', - description='These arguments are required if neither --machine nor --rebuild are given; ' - 'they are not allowed with either of those arguments:') + title="required arguments for a user-defined machine", + description="These arguments are required if neither --machine nor --rebuild are given; " + "they are not allowed with either of those arguments:", + ) new_machine_required_list = [] - new_machine_required.add_argument('--os', type=str.lower, - choices=['linux', 'aix', 'darwin', 'cnl'], - help='Operating system type') - new_machine_required_list.append('os') - - new_machine_required.add_argument('--netcdf-path', - help='Path to NetCDF installation\n' - '(path to top-level directory, containing subdirectories\n' - 'named lib, include, etc.)') - new_machine_required_list.append('netcdf-path') - - new_machine_required.add_argument('--esmf-mkfile-path', - help='Path to esmf.mk file\n' - '(typically within ESMF library directory)') - new_machine_required_list.append('esmf-mkfile-path') - - new_machine_required.add_argument('--max-mpitasks-per-node', type=int, - help='Number of physical processors per shared-memory node\n' - 'on this machine') - new_machine_required_list.append('max-mpitasks-per-node') + new_machine_required.add_argument( + "--os", + type=str.lower, + choices=["linux", "aix", "darwin", "cnl"], + help="Operating system type", + ) + new_machine_required_list.append("os") + + new_machine_required.add_argument( + "--netcdf-path", + help="Path to NetCDF installation\n" + "(path to top-level directory, containing subdirectories\n" + "named lib, include, etc.)", + ) + new_machine_required_list.append("netcdf-path") + + new_machine_required.add_argument( + "--esmf-mkfile-path", + help="Path to esmf.mk file\n" "(typically within ESMF library directory)", + ) + new_machine_required_list.append("esmf-mkfile-path") + + new_machine_required.add_argument( + "--max-mpitasks-per-node", + type=int, + help="Number of physical processors per shared-memory node\n" "on this machine", + ) + new_machine_required_list.append("max-mpitasks-per-node") new_machine_optional = parser.add_argument_group( - title='optional arguments for a user-defined machine', - description='These arguments are optional if neither --machine nor --rebuild are given; ' - 'they are not allowed with either of those arguments:') + title="optional arguments for a user-defined machine", + description="These arguments are optional if neither --machine nor --rebuild are given; " + "they are not allowed with either of those arguments:", + ) new_machine_optional_list = [] - new_machine_optional.add_argument('--gmake', default='gmake', - help='Name of GNU Make tool on your system\n' - 'Default: gmake') - new_machine_optional_list.append('gmake') - - new_machine_optional.add_argument('--gmake-j', default=8, type=int, - help='Number of threads to use when building\n' - 'Default: 8') - new_machine_optional_list.append('gmake-j') - - new_machine_optional.add_argument('--pnetcdf-path', - help='Path to PNetCDF installation, if present\n' - 'You must either specify this or set --no-pnetcdf') - new_machine_optional_list.append('pnetcdf-path') - - new_machine_optional.add_argument('--pio-filesystem-hints', type=str.lower, - choices=['gpfs', 'lustre'], - help='Enable filesystem hints for the given filesystem type\n' - 'when building the Parallel IO library') - new_machine_optional_list.append('pio-filesystem-hints') - - new_machine_optional.add_argument('--gptl-nano-timers', action='store_true', - help='Enable nano timers in build of the GPTL timing library') - new_machine_optional_list.append('gptl-nano-timers') - - new_machine_optional.add_argument('--extra-fflags', default='', - help='Any extra, non-standard flags to include\n' - 'when compiling Fortran files\n' - 'Tip: to allow a dash at the start of these flags,\n' - 'use a quoted string with an initial space, as in:\n' - ' --extra-fflags " -flag1 -flag2"') - new_machine_optional_list.append('extra-fflags') - - new_machine_optional.add_argument('--extra-cflags', default='', - help='Any extra, non-standard flags to include\n' - 'when compiling C files\n' - 'Tip: to allow a dash at the start of these flags,\n' - 'use a quoted string with an initial space, as in:\n' - ' --extra-cflags " -flag1 -flag2"') - new_machine_optional_list.append('extra-cflags') + new_machine_optional.add_argument( + "--gmake", + default="gmake", + help="Name of GNU Make tool on your system\n" "Default: gmake", + ) + new_machine_optional_list.append("gmake") + + new_machine_optional.add_argument( + "--gmake-j", + default=8, + type=int, + help="Number of threads to use when building\n" "Default: 8", + ) + new_machine_optional_list.append("gmake-j") + + new_machine_optional.add_argument( + "--pnetcdf-path", + help="Path to PNetCDF installation, if present\n" + "You must either specify this or set --no-pnetcdf", + ) + new_machine_optional_list.append("pnetcdf-path") + + new_machine_optional.add_argument( + "--pio-filesystem-hints", + type=str.lower, + choices=["gpfs", "lustre"], + help="Enable filesystem hints for the given filesystem type\n" + "when building the Parallel IO library", + ) + new_machine_optional_list.append("pio-filesystem-hints") + + new_machine_optional.add_argument( + "--gptl-nano-timers", + action="store_true", + help="Enable nano timers in build of the GPTL timing library", + ) + new_machine_optional_list.append("gptl-nano-timers") + + new_machine_optional.add_argument( + "--extra-fflags", + default="", + help="Any extra, non-standard flags to include\n" + "when compiling Fortran files\n" + "Tip: to allow a dash at the start of these flags,\n" + "use a quoted string with an initial space, as in:\n" + ' --extra-fflags " -flag1 -flag2"', + ) + new_machine_optional_list.append("extra-fflags") + + new_machine_optional.add_argument( + "--extra-cflags", + default="", + help="Any extra, non-standard flags to include\n" + "when compiling C files\n" + "Tip: to allow a dash at the start of these flags,\n" + "use a quoted string with an initial space, as in:\n" + ' --extra-cflags " -flag1 -flag2"', + ) + new_machine_optional_list.append("extra-cflags") add_logging_args(parser) args = parser.parse_args(args_to_parse) if args.rebuild: - _confirm_args_absent(parser, args, "cannot be provided if --rebuild is set", - (non_rebuild_required_list + non_rebuild_optional_list + - new_machine_required_list + new_machine_optional_list)) + _confirm_args_absent( + parser, + args, + "cannot be provided if --rebuild is set", + ( + non_rebuild_required_list + + non_rebuild_optional_list + + new_machine_required_list + + new_machine_optional_list + ), + ) else: - _confirm_args_present(parser, args, "must be provided if --rebuild is not set", - non_rebuild_required_list) + _confirm_args_present( + parser, + args, + "must be provided if --rebuild is not set", + non_rebuild_required_list, + ) if args.machine: - _confirm_args_absent(parser, args, "cannot be provided if --machine is set", - new_machine_required_list + new_machine_optional_list) + _confirm_args_absent( + parser, + args, + "cannot be provided if --machine is set", + new_machine_required_list + new_machine_optional_list, + ) else: - _confirm_args_present(parser, args, "must be provided if neither --machine nor --rebuild are set", - new_machine_required_list) + _confirm_args_present( + parser, + args, + "must be provided if neither --machine nor --rebuild are set", + new_machine_required_list, + ) if not args.no_pnetcdf and args.pnetcdf_path is None: - parser.error("For a user-defined machine, need to specify either --no-pnetcdf or --pnetcdf-path") + parser.error( + "For a user-defined machine, need to specify either --no-pnetcdf or --pnetcdf-path" + ) if args.no_pnetcdf and args.pnetcdf_path is not None: parser.error("--no-pnetcdf cannot be given if you set --pnetcdf-path") return args + def _confirm_args_absent(parser, args, errmsg, args_not_allowed): """Confirms that all args not allowed in this usage are absent @@ -438,13 +540,14 @@ def _confirm_args_absent(parser, args, errmsg, args_not_allowed): args_not_allowed: list of strings - argument names in this category """ for arg in args_not_allowed: - arg_no_dashes = arg.replace('-', '_') + arg_no_dashes = arg.replace("-", "_") # To determine whether the user specified an argument, we look at whether its # value differs from its default value. This won't catch the case where the user # explicitly set an argument to its default value, but it's not a big deal if we # miss printing an error in that case. if vars(args)[arg_no_dashes] != parser.get_default(arg_no_dashes): - parser.error('--{} {}'.format(arg, errmsg)) + parser.error("--{} {}".format(arg, errmsg)) + def _confirm_args_present(parser, args, errmsg, args_required): """Confirms that all args required in this usage are present @@ -458,28 +561,28 @@ def _confirm_args_present(parser, args, errmsg, args_required): args_required: list of strings - argument names in this category """ for arg in args_required: - arg_no_dashes = arg.replace('-', '_') + arg_no_dashes = arg.replace("-", "_") if vars(args)[arg_no_dashes] is None: - parser.error('--{} {}'.format(arg, errmsg)) + parser.error("--{} {}".format(arg, errmsg)) + def _check_and_transform_os(os_type): """Check validity of os_type argument and transform it to proper case os_type should be a lowercase string; returns a transformed string """ - transforms = {'linux': 'LINUX', - 'aix': 'AIX', - 'darwin': 'Darwin', - 'cnl': 'CNL'} + transforms = {"linux": "LINUX", "aix": "AIX", "darwin": "Darwin", "cnl": "CNL"} try: os_type_transformed = transforms[os_type] except KeyError as exc: raise ValueError("Unknown OS: {}".format(os_type)) from exc return os_type_transformed + def _get_case_dir(build_dir): """Given the path to build_dir, return the path to the case directory""" - return os.path.join(build_dir, 'case') + return os.path.join(build_dir, "case") + def _create_build_dir(build_dir, existing_inputdata): """Create the given build directory and any necessary sub-directories @@ -489,25 +592,30 @@ def _create_build_dir(build_dir, existing_inputdata): existing_inputdata (bool): whether the inputdata directory already exists on this machine """ if os.path.exists(build_dir): - abort('When running without --rebuild, the build directory must not exist yet\n' - '(<{}> already exists)'.format(build_dir)) + abort( + "When running without --rebuild, the build directory must not exist yet\n" + "(<{}> already exists)".format(build_dir) + ) os.makedirs(build_dir) if not existing_inputdata: os.makedirs(os.path.join(build_dir, _INPUTDATA_DIRNAME)) -def _fill_out_machine_files(build_dir, - os_type, - compiler, - netcdf_path, - esmf_mkfile_path, - max_mpitasks_per_node, - gmake, - gmake_j, - pnetcdf_path=None, - pio_filesystem_hints=None, - gptl_nano_timers=False, - extra_fflags='', - extra_cflags=''): + +def _fill_out_machine_files( + build_dir, + os_type, + compiler, + netcdf_path, + esmf_mkfile_path, + max_mpitasks_per_node, + gmake, + gmake_j, + pnetcdf_path=None, + pio_filesystem_hints=None, + gptl_nano_timers=False, + extra_fflags="", + extra_cflags="", +): """Fill out the machine porting templates for this machine / compiler For documentation of args, see the documentation in the build_ctsm function @@ -519,15 +627,22 @@ def _fill_out_machine_files(build_dir, # ------------------------------------------------------------------------ fill_template_file( - path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'config_machines_template.xml'), - path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, 'config_machines.xml'), - substitutions={'OS':os_type, - 'COMPILER':compiler, - 'CIME_OUTPUT_ROOT':build_dir, - 'GMAKE':gmake, - 'GMAKE_J':gmake_j, - 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node, - 'ESMF_MKFILE_PATH':esmf_mkfile_path}) + path_to_template=os.path.join( + _PATH_TO_TEMPLATES, "config_machines_template.xml" + ), + path_to_final=os.path.join( + build_dir, _MACHINE_CONFIG_DIRNAME, "config_machines.xml" + ), + substitutions={ + "OS": os_type, + "COMPILER": compiler, + "CIME_OUTPUT_ROOT": build_dir, + "GMAKE": gmake, + "GMAKE_J": gmake_j, + "MAX_MPITASKS_PER_NODE": max_mpitasks_per_node, + "ESMF_MKFILE_PATH": esmf_mkfile_path, + }, + ) # ------------------------------------------------------------------------ # Fill in ctsm-build_template.cmake @@ -536,36 +651,48 @@ def _fill_out_machine_files(build_dir, if gptl_nano_timers: gptl_cppdefs = _GPTL_NANOTIMERS_CPPDEFS else: - gptl_cppdefs = '' + gptl_cppdefs = "" if pio_filesystem_hints: pio_filesystem_hints_addition = 'set(PIO_FILESYSTEM_HINTS "{}")'.format( - pio_filesystem_hints) + pio_filesystem_hints + ) else: - pio_filesystem_hints_addition = '' + pio_filesystem_hints_addition = "" if pnetcdf_path: - pnetcdf_path_addition = 'set(PNETCDF_PATH "{}")'.format( - pnetcdf_path) + pnetcdf_path_addition = 'set(PNETCDF_PATH "{}")'.format(pnetcdf_path) else: - pnetcdf_path_addition = '' + pnetcdf_path_addition = "" fill_template_file( - path_to_template=os.path.join(_PATH_TO_TEMPLATES, - 'ctsm-build_template.cmake'), - path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros", - '{}_{}.cmake'.format(compiler, _MACH_NAME)), - substitutions={'GPTL_CPPDEFS':gptl_cppdefs, - 'NETCDF_PATH':netcdf_path, - 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_addition, - 'PNETCDF_PATH':pnetcdf_path_addition, - 'EXTRA_CFLAGS':extra_cflags, - 'EXTRA_FFLAGS':extra_fflags}) - - -def _create_case(cime_path, build_dir, compiler, - machine=None, build_debug=False, build_with_openmp=False, - inputdata_path=None): + path_to_template=os.path.join(_PATH_TO_TEMPLATES, "ctsm-build_template.cmake"), + path_to_final=os.path.join( + build_dir, + _MACHINE_CONFIG_DIRNAME, + "cmake_macros", + "{}_{}.cmake".format(compiler, _MACH_NAME), + ), + substitutions={ + "GPTL_CPPDEFS": gptl_cppdefs, + "NETCDF_PATH": netcdf_path, + "PIO_FILESYSTEM_HINTS": pio_filesystem_hints_addition, + "PNETCDF_PATH": pnetcdf_path_addition, + "EXTRA_CFLAGS": extra_cflags, + "EXTRA_FFLAGS": extra_fflags, + }, + ) + + +def _create_case( + cime_path, + build_dir, + compiler, + machine=None, + build_debug=False, + build_with_openmp=False, + inputdata_path=None, +): """Create a case that can later be used to build the CTSM library and its dependencies Args: @@ -591,49 +718,67 @@ def _create_case(cime_path, build_dir, compiler, # require you to be in the case directory when you execute them. case_dir = _get_case_dir(build_dir) - xmlchange = os.path.join(case_dir, 'xmlchange') + xmlchange = os.path.join(case_dir, "xmlchange") if machine is None: - machine_args = ['--machine', _MACH_NAME, - '--extra-machines-dir', os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME)] + machine_args = [ + "--machine", + _MACH_NAME, + "--extra-machines-dir", + os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME), + ] else: - machine_args = ['--machine', machine] - - create_newcase_cmd = [os.path.join(cime_path, 'scripts', 'create_newcase'), - '--output-root', build_dir, - '--case', case_dir, - '--compset', _COMPSET, - '--res', _RES, - '--compiler', compiler, - '--driver', 'nuopc', - # Project isn't used for anything in the LILAC workflow, but it - # still needs to be specified on machines that expect it. - '--project', 'UNSET', - '--run-unsupported'] + machine_args = ["--machine", machine] + + create_newcase_cmd = [ + os.path.join(cime_path, "scripts", "create_newcase"), + "--output-root", + build_dir, + "--case", + case_dir, + "--compset", + _COMPSET, + "--res", + _RES, + "--compiler", + compiler, + "--driver", + "nuopc", + # Project isn't used for anything in the LILAC workflow, but it + # still needs to be specified on machines that expect it. + "--project", + "UNSET", + "--run-unsupported", + ] create_newcase_cmd.extend(machine_args) if inputdata_path: - create_newcase_cmd.extend(['--input-dir', inputdata_path]) - run_cmd_output_on_error(create_newcase_cmd, - errmsg='Problem creating CTSM case directory') + create_newcase_cmd.extend(["--input-dir", inputdata_path]) + run_cmd_output_on_error( + create_newcase_cmd, errmsg="Problem creating CTSM case directory" + ) - subprocess.check_call([xmlchange, 'LILAC_MODE=on'], cwd=case_dir) + subprocess.check_call([xmlchange, "LILAC_MODE=on"], cwd=case_dir) if build_debug: - subprocess.check_call([xmlchange, 'DEBUG=TRUE'], cwd=case_dir) + subprocess.check_call([xmlchange, "DEBUG=TRUE"], cwd=case_dir) if build_with_openmp: - subprocess.check_call([xmlchange, 'FORCE_BUILD_SMP=TRUE'], cwd=case_dir) + subprocess.check_call([xmlchange, "FORCE_BUILD_SMP=TRUE"], cwd=case_dir) - run_cmd_output_on_error([os.path.join(case_dir, 'case.setup')], - errmsg='Problem setting up CTSM case directory', - cwd=case_dir) + run_cmd_output_on_error( + [os.path.join(case_dir, "case.setup")], + errmsg="Problem setting up CTSM case directory", + cwd=case_dir, + ) - make_link(os.path.join(case_dir, 'bld'), - os.path.join(build_dir, 'bld')) + make_link(os.path.join(case_dir, "bld"), os.path.join(build_dir, "bld")) if machine is not None: # For a pre-existing machine, the .env_mach_specific files are likely useful to # the user. Make sym links to these with more intuitive names. - for extension in ('sh', 'csh'): - make_link(os.path.join(case_dir, '.env_mach_specific.{}'.format(extension)), - os.path.join(build_dir, 'ctsm_build_environment.{}'.format(extension))) + for extension in ("sh", "csh"): + make_link( + os.path.join(case_dir, ".env_mach_specific.{}".format(extension)), + os.path.join(build_dir, "ctsm_build_environment.{}".format(extension)), + ) + def _stage_runtime_inputs(build_dir, no_pnetcdf): """Stage CTSM and LILAC runtime inputs @@ -644,44 +789,58 @@ def _stage_runtime_inputs(build_dir, no_pnetcdf): """ os.makedirs(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME)) - inputdata_dir = _xmlquery('DIN_LOC_ROOT', build_dir) + inputdata_dir = _xmlquery("DIN_LOC_ROOT", build_dir) fill_template_file( - path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'ctsm_template.cfg'), - path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'ctsm.cfg'), - substitutions={'INPUTDATA':inputdata_dir}) + path_to_template=os.path.join(_PATH_TO_TEMPLATES, "ctsm_template.cfg"), + path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, "ctsm.cfg"), + substitutions={"INPUTDATA": inputdata_dir}, + ) fill_template_file( - path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'lilac_in_template'), - path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'lilac_in'), - substitutions={'INPUTDATA':inputdata_dir}) + path_to_template=os.path.join(_PATH_TO_TEMPLATES, "lilac_in_template"), + path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, "lilac_in"), + substitutions={"INPUTDATA": inputdata_dir}, + ) - pio_stride = _xmlquery('MAX_MPITASKS_PER_NODE', build_dir) + pio_stride = _xmlquery("MAX_MPITASKS_PER_NODE", build_dir) if no_pnetcdf: - pio_typename = 'netcdf' + pio_typename = "netcdf" # pio_rearranger = 1 is generally more efficient with netcdf (see # https://github.com/ESMCI/cime/pull/3732#discussion_r508954806 and the following # discussion) pio_rearranger = 1 else: - pio_typename = 'pnetcdf' + pio_typename = "pnetcdf" pio_rearranger = 2 fill_template_file( - path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'lnd_modelio_template.nml'), - path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'lnd_modelio.nml'), - substitutions={'PIO_REARRANGER':pio_rearranger, - 'PIO_STRIDE':pio_stride, - 'PIO_TYPENAME':pio_typename}) + path_to_template=os.path.join(_PATH_TO_TEMPLATES, "lnd_modelio_template.nml"), + path_to_final=os.path.join( + build_dir, _RUNTIME_INPUTS_DIRNAME, "lnd_modelio.nml" + ), + substitutions={ + "PIO_REARRANGER": pio_rearranger, + "PIO_STRIDE": pio_stride, + "PIO_TYPENAME": pio_typename, + }, + ) shutil.copyfile( - src=os.path.join(path_to_ctsm_root(), - 'cime_config', 'usermods_dirs', 'lilac', 'user_nl_ctsm'), - dst=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'user_nl_ctsm')) + src=os.path.join( + path_to_ctsm_root(), "cime_config", "usermods_dirs", "lilac", "user_nl_ctsm" + ), + dst=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, "user_nl_ctsm"), + ) - make_link(_PATH_TO_MAKE_RUNTIME_INPUTS, - os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'make_runtime_inputs')) + make_link( + _PATH_TO_MAKE_RUNTIME_INPUTS, + os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, "make_runtime_inputs"), + ) + + make_link( + _PATH_TO_DOWNLOAD_INPUT_DATA, + os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, "download_input_data"), + ) - make_link(_PATH_TO_DOWNLOAD_INPUT_DATA, - os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'download_input_data')) def _build_case(build_dir): """Build the CTSM library and its dependencies @@ -697,20 +856,21 @@ def _build_case(build_dir): case_dir = _get_case_dir(build_dir) try: subprocess.check_call( - [os.path.join(case_dir, 'case.build'), - '--sharedlib-only'], - cwd=case_dir) + [os.path.join(case_dir, "case.build"), "--sharedlib-only"], cwd=case_dir + ) except subprocess.CalledProcessError: - abort('ERROR building CTSM or its dependencies - see above for details') + abort("ERROR building CTSM or its dependencies - see above for details") + + make_link( + os.path.join(case_dir, "bld", "ctsm.mk"), os.path.join(build_dir, "ctsm.mk") + ) - make_link(os.path.join(case_dir, 'bld', 'ctsm.mk'), - os.path.join(build_dir, 'ctsm.mk')) def _xmlquery(varname, build_dir): """Run xmlquery from the case in build_dir and return the value of the given variable""" case_dir = _get_case_dir(build_dir) - xmlquery_path = os.path.join(case_dir, 'xmlquery') - value = subprocess.check_output([xmlquery_path, '--value', varname], - cwd=case_dir, - universal_newlines=True) + xmlquery_path = os.path.join(case_dir, "xmlquery") + value = subprocess.check_output( + [xmlquery_path, "--value", varname], cwd=case_dir, universal_newlines=True + ) return value diff --git a/python/ctsm/lilac_download_input_data.py b/python/ctsm/lilac_download_input_data.py index dffc78150a..90ea5ce23f 100644 --- a/python/ctsm/lilac_download_input_data.py +++ b/python/ctsm/lilac_download_input_data.py @@ -5,9 +5,13 @@ import os import re -from CIME.case import Case # pylint: disable=import-error +from CIME.case import Case # pylint: disable=import-error -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) logger = logging.getLogger(__name__) @@ -23,15 +27,16 @@ # Public functions # ======================================================================== + def main(): - """Main function called when download_input_data is run from the command-line - """ + """Main function called when download_input_data is run from the command-line""" setup_logging_pre_config() args = _commandline_args() process_logging_args(args) download_input_data(rundir=args.rundir) + def download_input_data(rundir): """Implementation of the download_input_data command @@ -39,35 +44,36 @@ def download_input_data(rundir): rundir: str - path to directory containing input_data_list files """ _create_lilac_input_data_list(rundir) - case = Case(os.path.realpath(os.path.join(rundir, os.pardir, 'case'))) - case.check_all_input_data( - data_list_dir=rundir, - download=True, - chksum=False) - os.remove(os.path.join(rundir, 'lilac.input_data_list')) + case = Case(os.path.realpath(os.path.join(rundir, os.pardir, "case"))) + case.check_all_input_data(data_list_dir=rundir, download=True, chksum=False) + os.remove(os.path.join(rundir, "lilac.input_data_list")) + # ======================================================================== # Private functions # ======================================================================== + def _commandline_args(): - """Parse and return command-line arguments - """ + """Parse and return command-line arguments""" description = """ Script to download any missing input data for CTSM and LILAC """ parser = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter) - - parser.add_argument("--rundir", default=os.getcwd(), - help="Full path of the run directory\n" - "(This directory should contain clm.input_data_list and lilac_in,\n" - "among other files.)\n" - "(Note: it is assumed that this directory exists alongside the other\n" - "directories created by build_ctsm: 'case' and 'inputdata'.)") + description=description, formatter_class=argparse.RawTextHelpFormatter + ) + + parser.add_argument( + "--rundir", + default=os.getcwd(), + help="Full path of the run directory\n" + "(This directory should contain clm.input_data_list and lilac_in,\n" + "among other files.)\n" + "(Note: it is assumed that this directory exists alongside the other\n" + "directories created by build_ctsm: 'case' and 'inputdata'.)", + ) add_logging_args(parser) @@ -75,12 +81,15 @@ def _commandline_args(): return args + def _create_lilac_input_data_list(rundir): - with open(os.path.join(rundir, 'lilac_in')) as lilac_in: - with open(os.path.join(rundir, 'lilac.input_data_list'), 'w') as input_data_list: + with open(os.path.join(rundir, "lilac_in")) as lilac_in: + with open( + os.path.join(rundir, "lilac.input_data_list"), "w" + ) as input_data_list: for line in lilac_in: if re.search(_LILAC_FILENAME, line): # Remove quotes from filename, then output this line - line = line.replace('"', '') + line = line.replace('"', "") line = line.replace("'", "") input_data_list.write(line) diff --git a/python/ctsm/lilac_make_runtime_inputs.py b/python/ctsm/lilac_make_runtime_inputs.py index 1e6f16fb4e..c82fb8a048 100644 --- a/python/ctsm/lilac_make_runtime_inputs.py +++ b/python/ctsm/lilac_make_runtime_inputs.py @@ -8,9 +8,13 @@ from configparser import ConfigParser from configparser import NoSectionError, NoOptionError -from CIME.buildnml import create_namelist_infile # pylint: disable=import-error +from CIME.buildnml import create_namelist_infile # pylint: disable=import-error -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) from ctsm.path_utils import path_to_ctsm_root from ctsm.utils import abort @@ -31,7 +35,7 @@ # Note the following is needed in env_lilac.xml otherwise the following error appears in # the call to build_namelist -#err=ERROR : CLM build-namelist::CLMBuildNamelist::logical_to_fortran() : +# err=ERROR : CLM build-namelist::CLMBuildNamelist::logical_to_fortran() : # Unexpected value in logical_to_fortran: _ENV_LILAC_TEMPLATE = """ @@ -48,19 +52,21 @@ # This string is used in the out-of-the-box ctsm.cfg file to denote a value that needs to # be filled in -_PLACEHOLDER = 'FILL_THIS_IN' +_PLACEHOLDER = "FILL_THIS_IN" # This string is used in the out-of-the-box ctsm.cfg file to denote a value that can be # filled in, but doesn't absolutely need to be -_UNSET = 'UNSET' +_UNSET = "UNSET" # ======================================================================== # Fake case class that can be used to satisfy the interface of CIME functions that need a # case object # ======================================================================== + class CaseFake: """Fake case class to satisfy interface of CIME functions that need a case object""" + # pylint: disable=too-few-public-methods def __init__(self): @@ -74,9 +80,10 @@ def get_resolved_value(value): """ abort("Cannot resolve value with a '$' variable: {}".format(value)) + ############################################################################### def parse_command_line(): -############################################################################### + ############################################################################### """Parse the command line, return object holding arguments""" @@ -84,11 +91,16 @@ def parse_command_line(): Script to create runtime inputs when running CTSM via LILAC """ - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, - description=description) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, description=description + ) - parser.add_argument("--rundir", type=str, default=os.getcwd(), - help="Full path of the run directory (containing ctsm.cfg & user_nl_ctsm)") + parser.add_argument( + "--rundir", + type=str, + default=os.getcwd(), + help="Full path of the run directory (containing ctsm.cfg & user_nl_ctsm)", + ) add_logging_args(parser) @@ -101,6 +113,7 @@ def parse_command_line(): return arguments + ############################################################################### def get_config_value(config, section, item, file_path, allowed_values=None): """Get a given item from a given section of the config object @@ -114,106 +127,153 @@ def get_config_value(config, section, item, file_path, allowed_values=None): try: val = config.get(section, item) except NoSectionError: - abort("ERROR: Config file {} must contain section '{}'".format(file_path, section)) + abort( + "ERROR: Config file {} must contain section '{}'".format(file_path, section) + ) except NoOptionError: - abort("ERROR: Config file {} must contain item '{}' in section '{}'".format( - file_path, item, section)) + abort( + "ERROR: Config file {} must contain item '{}' in section '{}'".format( + file_path, item, section + ) + ) if val == _PLACEHOLDER: - abort("Error: {} needs to be specified in config file {}".format(item, file_path)) + abort( + "Error: {} needs to be specified in config file {}".format(item, file_path) + ) if allowed_values is not None: if val not in allowed_values: - abort("Error: {} is not an allowed value for {} in config file {}\n" - "Allowed values: {}".format(val, item, file_path, allowed_values)) + abort( + "Error: {} is not an allowed value for {} in config file {}\n" + "Allowed values: {}".format(val, item, file_path, allowed_values) + ) return val + ############################################################################### def determine_bldnml_opts(bgc_mode, crop, vichydro): -############################################################################### + ############################################################################### """Return a string giving bldnml options, given some other inputs""" - bldnml_opts = '' - bldnml_opts += ' -bgc {}'.format(bgc_mode) - if bgc_mode == 'fates': + bldnml_opts = "" + bldnml_opts += " -bgc {}".format(bgc_mode) + if bgc_mode == "fates": # BUG(wjs, 2020-06-12, ESCOMP/CTSM#115) For now, FATES is incompatible with MEGAN - bldnml_opts += ' -no-megan' - - if crop == 'on': - if bgc_mode not in ['bgc', 'cn']: - abort("Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'") - bldnml_opts += ' -crop' - - if vichydro == 'on': - if bgc_mode != 'sp': - abort("Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'") - bldnml_opts += ' -vichydro' + bldnml_opts += " -no-megan" + + if crop == "on": + if bgc_mode not in ["bgc", "cn"]: + abort( + "Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'" + ) + bldnml_opts += " -crop" + + if vichydro == "on": + if bgc_mode != "sp": + abort( + "Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'" + ) + bldnml_opts += " -vichydro" return bldnml_opts + ############################################################################### def buildnml(cime_path, rundir): -############################################################################### + ############################################################################### - """Build the ctsm namelist - """ + """Build the ctsm namelist""" # pylint: disable=too-many-locals # pylint: disable=too-many-statements - ctsm_cfg_path = os.path.join(rundir, 'ctsm.cfg') + ctsm_cfg_path = os.path.join(rundir, "ctsm.cfg") # read the config file config = ConfigParser() config.read(ctsm_cfg_path) - lnd_domain_file = get_config_value(config, 'buildnml_input', 'lnd_domain_file', ctsm_cfg_path) - fsurdat = get_config_value(config, 'buildnml_input', 'fsurdat', ctsm_cfg_path) - finidat = get_config_value(config, 'buildnml_input', 'finidat', ctsm_cfg_path) - - ctsm_phys = get_config_value(config, 'buildnml_input', 'ctsm_phys', ctsm_cfg_path, - allowed_values=['clm4_5', 'clm5_0', 'clm5_1']) - configuration = get_config_value(config, 'buildnml_input', 'configuration', ctsm_cfg_path, - allowed_values=['nwp', 'clm']) - structure = get_config_value(config, 'buildnml_input', 'structure', ctsm_cfg_path, - allowed_values=['fast', 'standard']) - bgc_mode = get_config_value(config, 'buildnml_input', 'bgc_mode', ctsm_cfg_path, - allowed_values=['sp', 'bgc', 'cn', 'fates']) - crop = get_config_value(config, 'buildnml_input', 'crop', ctsm_cfg_path, - allowed_values=['off', 'on']) - vichydro = get_config_value(config, 'buildnml_input', 'vichydro', ctsm_cfg_path, - allowed_values=['off', 'on']) - - bldnml_opts = determine_bldnml_opts(bgc_mode=bgc_mode, - crop=crop, - vichydro=vichydro) - - co2_ppmv = get_config_value(config, 'buildnml_input', 'co2_ppmv', ctsm_cfg_path) - use_case = get_config_value(config, 'buildnml_input', 'use_case', ctsm_cfg_path) - lnd_tuning_mode = get_config_value(config, 'buildnml_input', 'lnd_tuning_mode', ctsm_cfg_path) - spinup = get_config_value(config, 'buildnml_input', 'spinup', ctsm_cfg_path, - allowed_values=['off', 'on']) - - inputdata_path = get_config_value(config, 'buildnml_input', 'inputdata_path', ctsm_cfg_path) + lnd_domain_file = get_config_value( + config, "buildnml_input", "lnd_domain_file", ctsm_cfg_path + ) + fsurdat = get_config_value(config, "buildnml_input", "fsurdat", ctsm_cfg_path) + finidat = get_config_value(config, "buildnml_input", "finidat", ctsm_cfg_path) + + ctsm_phys = get_config_value( + config, + "buildnml_input", + "ctsm_phys", + ctsm_cfg_path, + allowed_values=["clm4_5", "clm5_0", "clm5_1"], + ) + configuration = get_config_value( + config, + "buildnml_input", + "configuration", + ctsm_cfg_path, + allowed_values=["nwp", "clm"], + ) + structure = get_config_value( + config, + "buildnml_input", + "structure", + ctsm_cfg_path, + allowed_values=["fast", "standard"], + ) + bgc_mode = get_config_value( + config, + "buildnml_input", + "bgc_mode", + ctsm_cfg_path, + allowed_values=["sp", "bgc", "cn", "fates"], + ) + crop = get_config_value( + config, "buildnml_input", "crop", ctsm_cfg_path, allowed_values=["off", "on"] + ) + vichydro = get_config_value( + config, + "buildnml_input", + "vichydro", + ctsm_cfg_path, + allowed_values=["off", "on"], + ) + + bldnml_opts = determine_bldnml_opts(bgc_mode=bgc_mode, crop=crop, vichydro=vichydro) + + co2_ppmv = get_config_value(config, "buildnml_input", "co2_ppmv", ctsm_cfg_path) + use_case = get_config_value(config, "buildnml_input", "use_case", ctsm_cfg_path) + lnd_tuning_mode = get_config_value( + config, "buildnml_input", "lnd_tuning_mode", ctsm_cfg_path + ) + spinup = get_config_value( + config, "buildnml_input", "spinup", ctsm_cfg_path, allowed_values=["off", "on"] + ) + + inputdata_path = get_config_value( + config, "buildnml_input", "inputdata_path", ctsm_cfg_path + ) # Parse the user_nl_ctsm file - infile = os.path.join(rundir, '.namelist') - create_namelist_infile(case=CaseFake(), - user_nl_file=os.path.join(rundir, 'user_nl_ctsm'), - namelist_infile=infile) + infile = os.path.join(rundir, ".namelist") + create_namelist_infile( + case=CaseFake(), + user_nl_file=os.path.join(rundir, "user_nl_ctsm"), + namelist_infile=infile, + ) # create config_cache.xml file # Note that build-namelist utilizes the contents of the config_cache.xml file in # the namelist_defaults.xml file to obtain namelist variables config_cache = os.path.join(rundir, "config_cache.xml") config_cache_text = _CONFIG_CACHE_TEMPLATE.format(clm_phys=ctsm_phys) - with open(config_cache, 'w') as tempfile: + with open(config_cache, "w") as tempfile: tempfile.write(config_cache_text) # create temporary env_lilac.xml env_lilac = os.path.join(rundir, "env_lilac.xml") env_lilac_text = _ENV_LILAC_TEMPLATE.format() - with open(env_lilac, 'w') as tempfile: + with open(env_lilac, "w") as tempfile: tempfile.write(env_lilac_text) # remove any existing clm.input_data_list file @@ -222,7 +282,7 @@ def buildnml(cime_path, rundir): os.remove(inputdatalist_path) # determine if fsurdat and/or finidat should appear in the -namelist option - extra_namelist_opts = '' + extra_namelist_opts = "" if fsurdat != _UNSET: # NOTE(wjs, 2020-06-30) With the current logic, fsurdat should never be _UNSET, # but it's possible that this will change in the future. @@ -232,45 +292,63 @@ def buildnml(cime_path, rundir): # call build-namelist cmd = os.path.abspath(os.path.join(path_to_ctsm_root(), "bld", "build-namelist")) - command = [cmd, - '-driver', 'nuopc', - '-cimeroot', cime_path, - '-infile', infile, - '-csmdata', inputdata_path, - '-inputdata', inputdatalist_path, - # Hard-code start_ymd of year-2000. This is used to set the run type (for - # which a setting of 2000 gives 'startup', which is what we want) and pick - # the initial conditions file (which is pretty much irrelevant when running - # with lilac). - '-namelist', '&clm_inparm start_ymd=20000101 {} /'.format(extra_namelist_opts), - '-use_case', use_case, - # For now, we assume ignore_ic_year, not ignore_ic_date - '-ignore_ic_year', - # -clm_start_type seems unimportant (see discussion in - # https://github.com/ESCOMP/CTSM/issues/876) - '-clm_start_type', 'default', - '-configuration', configuration, - '-structure', structure, - '-lilac', - '-lnd_frac', lnd_domain_file, - '-glc_nec', str(10), - '-co2_ppmv', co2_ppmv, - '-co2_type', 'constant', - '-clm_accelerated_spinup', spinup, - '-lnd_tuning_mode', lnd_tuning_mode, - # Eventually make -no-megan dynamic (see - # https://github.com/ESCOMP/CTSM/issues/926) - '-no-megan', - '-config', os.path.join(rundir, "config_cache.xml"), - '-envxml_dir', rundir] + command = [ + cmd, + "-driver", + "nuopc", + "-cimeroot", + cime_path, + "-infile", + infile, + "-csmdata", + inputdata_path, + "-inputdata", + inputdatalist_path, + # Hard-code start_ymd of year-2000. This is used to set the run type (for + # which a setting of 2000 gives 'startup', which is what we want) and pick + # the initial conditions file (which is pretty much irrelevant when running + # with lilac). + "-namelist", + "&clm_inparm start_ymd=20000101 {} /".format(extra_namelist_opts), + "-use_case", + use_case, + # For now, we assume ignore_ic_year, not ignore_ic_date + "-ignore_ic_year", + # -clm_start_type seems unimportant (see discussion in + # https://github.com/ESCOMP/CTSM/issues/876) + "-clm_start_type", + "default", + "-configuration", + configuration, + "-structure", + structure, + "-lilac", + "-lnd_frac", + lnd_domain_file, + "-glc_nec", + str(10), + "-co2_ppmv", + co2_ppmv, + "-co2_type", + "constant", + "-clm_accelerated_spinup", + spinup, + "-lnd_tuning_mode", + lnd_tuning_mode, + # Eventually make -no-megan dynamic (see + # https://github.com/ESCOMP/CTSM/issues/926) + "-no-megan", + "-config", + os.path.join(rundir, "config_cache.xml"), + "-envxml_dir", + rundir, + ] # NOTE(wjs, 2020-06-16) Note that we do NOT use the -mask argument; it's possible that # we should be using it in some circumstances (I haven't looked into how it's used). - command.extend(['-res', 'lilac', - '-clm_usr_name', 'lilac']) + command.extend(["-res", "lilac", "-clm_usr_name", "lilac"]) command.extend(bldnml_opts.split()) - subprocess.check_call(command, - universal_newlines=True) + subprocess.check_call(command, universal_newlines=True) # remove temporary files in rundir os.remove(os.path.join(rundir, "config_cache.xml")) @@ -278,6 +356,7 @@ def buildnml(cime_path, rundir): os.remove(os.path.join(rundir, "drv_flds_in")) os.remove(infile) + ############################################################################### def main(cime_path): """Main function @@ -292,8 +371,7 @@ def main(cime_path): args = parse_command_line() process_logging_args(args) - buildnml( - cime_path=cime_path, - rundir=args.rundir) + buildnml(cime_path=cime_path, rundir=args.rundir) + ############################################################################### diff --git a/python/ctsm/machine.py b/python/ctsm/machine.py index 36e5c61788..6d683c73a2 100644 --- a/python/ctsm/machine.py +++ b/python/ctsm/machine.py @@ -4,8 +4,10 @@ import logging from collections import namedtuple from CIME.utils import get_project # pylint: disable=import-error -from ctsm.joblauncher.job_launcher_factory import \ - create_job_launcher, JOB_LAUNCHER_NOBATCH +from ctsm.joblauncher.job_launcher_factory import ( + create_job_launcher, + JOB_LAUNCHER_NOBATCH, +) logger = logging.getLogger(__name__) @@ -23,19 +25,31 @@ # user of the machine object to check for that possibility if need be. # # Similar notes apply to baseline_dir. -Machine = namedtuple('Machine', ['name', # str - 'scratch_dir', # str - 'baseline_dir', # str - 'account', # str or None - 'create_test_retry', # int - 'job_launcher']) # subclass of JobLauncherBase - -def create_machine(machine_name, defaults, job_launcher_type=None, - scratch_dir=None, account=None, - job_launcher_queue=None, job_launcher_walltime=None, - job_launcher_nice_level=None, - job_launcher_extra_args=None, - allow_missing_entries=False): +Machine = namedtuple( + "Machine", + [ + "name", # str + "scratch_dir", # str + "baseline_dir", # str + "account", # str or None + "create_test_retry", # int + "job_launcher", + ], +) # subclass of JobLauncherBase + + +def create_machine( + machine_name, + defaults, + job_launcher_type=None, + scratch_dir=None, + account=None, + job_launcher_queue=None, + job_launcher_walltime=None, + job_launcher_nice_level=None, + job_launcher_extra_args=None, + allow_missing_entries=False, +): """Create a machine object (of type Machine, as given above) This uses the provided (non-None) arguments to override any defaults provided via the @@ -99,14 +113,20 @@ def create_machine(machine_name, defaults, job_launcher_type=None, # machine object: this will always give the default value for this machine, and # other mechanisms will be given for overriding this in a particular case. create_test_retry = mach_defaults.create_test_retry - if account is None and mach_defaults.account_required and not allow_missing_entries: + if ( + account is None + and mach_defaults.account_required + and not allow_missing_entries + ): raise RuntimeError("Could not find an account code") else: if not allow_missing_entries: # This isn't exactly a missing entry, but the times we don't care about this # warning tend to be the same as the times when allow_missing_entries is true - logger.warning("machine %s not recognized; using generic no-batch settings", - machine_name) + logger.warning( + "machine %s not recognized; using generic no-batch settings", + machine_name, + ) if job_launcher_type is None: job_launcher_type = JOB_LAUNCHER_NOBATCH @@ -118,7 +138,7 @@ def create_machine(machine_name, defaults, job_launcher_type=None, # sense to have these in the argument list for this function: they should only come # from the defaults structure. If the user wants to provide their own arguments, those # should be provided via extra_args. - job_launcher_required_args = '' + job_launcher_required_args = "" if mach_defaults is not None: these_defaults = mach_defaults.job_launcher_defaults.get(job_launcher_type) @@ -135,21 +155,26 @@ def create_machine(machine_name, defaults, job_launcher_type=None, # Create the job launcher and the full machine object # ------------------------------------------------------------------------ - job_launcher = create_job_launcher(job_launcher_type=job_launcher_type, - account=account, - queue=job_launcher_queue, - walltime=job_launcher_walltime, - nice_level=job_launcher_nice_level, - required_args=job_launcher_required_args, - extra_args=job_launcher_extra_args, - allow_missing_entries=allow_missing_entries) - - return Machine(name=machine_name, - scratch_dir=scratch_dir, - baseline_dir=baseline_dir, - account=account, - create_test_retry=create_test_retry, - job_launcher=job_launcher) + job_launcher = create_job_launcher( + job_launcher_type=job_launcher_type, + account=account, + queue=job_launcher_queue, + walltime=job_launcher_walltime, + nice_level=job_launcher_nice_level, + required_args=job_launcher_required_args, + extra_args=job_launcher_extra_args, + allow_missing_entries=allow_missing_entries, + ) + + return Machine( + name=machine_name, + scratch_dir=scratch_dir, + baseline_dir=baseline_dir, + account=account, + create_test_retry=create_test_retry, + job_launcher=job_launcher, + ) + def get_possibly_overridden_mach_value(machine, varname, value=None): """Get the value to use for the given machine variable @@ -166,6 +191,7 @@ def get_possibly_overridden_mach_value(machine, varname, value=None): return value return getattr(machine, varname) + def _get_account(): account = get_project() return account diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index c6b624b885..02066b88fd 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -4,16 +4,20 @@ from collections import namedtuple import os -from ctsm.joblauncher.job_launcher_factory import \ - JOB_LAUNCHER_QSUB +from ctsm.joblauncher.job_launcher_factory import JOB_LAUNCHER_QSUB from ctsm.machine_utils import get_user -MachineDefaults = namedtuple('MachineDefaults', ['job_launcher_type', - 'scratch_dir', - 'baseline_dir', - 'account_required', - 'create_test_retry', - 'job_launcher_defaults']) +MachineDefaults = namedtuple( + "MachineDefaults", + [ + "job_launcher_type", + "scratch_dir", + "baseline_dir", + "account_required", + "create_test_retry", + "job_launcher_defaults", + ], +) # job_launcher_type: one of the JOB_LAUNCHERs defined in job_launcher_factory # scratch_dir: str # baseline_dir: str: The standard location for CTSM baselines on this machine @@ -31,55 +35,61 @@ # have defaults for qsub, because other launchers (like no_batch) don't need any # arguments. -QsubDefaults = namedtuple('QsubDefaults', ['queue', - 'walltime', - 'extra_args', - 'required_args']) +QsubDefaults = namedtuple( + "QsubDefaults", ["queue", "walltime", "extra_args", "required_args"] +) MACHINE_DEFAULTS = { - 'cheyenne': MachineDefaults( + "cheyenne": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir=os.path.join(os.path.sep, 'glade', 'scratch', get_user()), - baseline_dir=os.path.join(os.path.sep, 'glade', 'p', 'cgd', 'tss', 'ctsm_baselines'), + scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), + baseline_dir=os.path.join( + os.path.sep, "glade", "p", "cgd", "tss", "ctsm_baselines" + ), account_required=True, create_test_retry=0, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='regular', - walltime='11:50:00', - extra_args='', + queue="regular", + walltime="11:50:00", + extra_args="", # The following assumes a single node, with a single mpi proc; we may want # to add more flexibility in the future, making the node / proc counts # individually selectable - required_args= - '-l select=1:ncpus=36:mpiprocs=1 -V -r n -l inception=login -k oed') - }), - 'hobart': MachineDefaults( + required_args="-l select=1:ncpus=36:mpiprocs=1 -V -r n -l inception=login -k oed", + ) + }, + ), + "hobart": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir=os.path.join(os.path.sep, 'scratch', 'cluster', get_user()), - baseline_dir=os.path.join(os.path.sep, 'fs', 'cgd', 'csm', 'ccsm_baselines'), + scratch_dir=os.path.join(os.path.sep, "scratch", "cluster", get_user()), + baseline_dir=os.path.join(os.path.sep, "fs", "cgd", "csm", "ccsm_baselines"), account_required=False, create_test_retry=0, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='medium', - walltime='04:00:00', - extra_args='', - required_args='-l nodes=1:ppn=48 -r n') - }), - 'izumi': MachineDefaults( + queue="medium", + walltime="04:00:00", + extra_args="", + required_args="-l nodes=1:ppn=48 -r n", + ) + }, + ), + "izumi": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir=os.path.join(os.path.sep, 'scratch', 'cluster', get_user()), - baseline_dir=os.path.join(os.path.sep, 'fs', 'cgd', 'csm', 'ccsm_baselines'), + scratch_dir=os.path.join(os.path.sep, "scratch", "cluster", get_user()), + baseline_dir=os.path.join(os.path.sep, "fs", "cgd", "csm", "ccsm_baselines"), account_required=False, # jobs on izumi experience a high frequency of failures, often at the very end of # the job; so we'll automatically retry a failed job twice before giving up on it create_test_retry=2, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='medium', - walltime='04:00:00', - extra_args='', - required_args='-l nodes=1:ppn=48 -r n') - }) + queue="medium", + walltime="04:00:00", + extra_args="", + required_args="-l nodes=1:ppn=48 -r n", + ) + }, + ), } diff --git a/python/ctsm/machine_utils.py b/python/ctsm/machine_utils.py index 41459ce3de..223a6aa5da 100644 --- a/python/ctsm/machine_utils.py +++ b/python/ctsm/machine_utils.py @@ -11,20 +11,24 @@ # Public functions # ======================================================================== + def get_user(): """Return the current user name (string)""" return getpass.getuser() + def get_machine_name(): """Return the current machine name (string)""" full_hostname = socket.gethostname() - hostname = full_hostname.split('.')[0] + hostname = full_hostname.split(".")[0] return _machine_from_hostname(hostname) + # ======================================================================== # Private functions # ======================================================================== + def _machine_from_hostname(hostname): """Given a hostname (string), return the machine name (string) @@ -32,8 +36,8 @@ def _machine_from_hostname(hostname): be extended if there are other machines with special translation rules from hostname to machine name. """ - if re.match(r'cheyenne\d+', hostname): - machine = 'cheyenne' + if re.match(r"cheyenne\d+", hostname): + machine = "cheyenne" else: machine = hostname diff --git a/python/ctsm/os_utils.py b/python/ctsm/os_utils.py index aac5680057..8dad6136ca 100644 --- a/python/ctsm/os_utils.py +++ b/python/ctsm/os_utils.py @@ -5,6 +5,7 @@ import subprocess from ctsm.utils import abort + def run_cmd_output_on_error(cmd, errmsg, cwd=None): """Run the given command; suppress output but print it if there is an error @@ -17,26 +18,26 @@ def run_cmd_output_on_error(cmd, errmsg, cwd=None): cwd: string or None - path from which the command should be run """ try: - _ = subprocess.check_output(cmd, - stderr=subprocess.STDOUT, - universal_newlines=True, - cwd=cwd) + _ = subprocess.check_output( + cmd, stderr=subprocess.STDOUT, universal_newlines=True, cwd=cwd + ) except subprocess.CalledProcessError as error: - print('ERROR while running:') - print(' '.join(cmd)) + print("ERROR while running:") + print(" ".join(cmd)) if cwd is not None: - print('From {}'.format(cwd)) - print('') + print("From {}".format(cwd)) + print("") print(error.output) - print('') + print("") abort(errmsg) except: - print('ERROR trying to run:') - print(' '.join(cmd)) + print("ERROR trying to run:") + print(" ".join(cmd)) if cwd is not None: - print('From {}'.format(cwd)) + print("From {}".format(cwd)) raise + def make_link(src, dst): """Makes a link pointing to src named dst diff --git a/python/ctsm/path_utils.py b/python/ctsm/path_utils.py index 7972fcd297..7cc68a39a2 100644 --- a/python/ctsm/path_utils.py +++ b/python/ctsm/path_utils.py @@ -14,23 +14,27 @@ # # Note: It's important that this NOT end with a trailing slash; # os.path.normpath guarantees this. -_CTSM_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, - os.pardir)) +_CTSM_ROOT = os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir) +) # Candidates for the last path components to the CTSM directory within a # CESM checkout -_CESM_CTSM_PATHS = [os.path.join('components', 'ctsm'), - os.path.join('components', 'clm')] +_CESM_CTSM_PATHS = [ + os.path.join("components", "ctsm"), + os.path.join("components", "clm"), +] # ======================================================================== # Public functions # ======================================================================== + def path_to_ctsm_root(): """Returns the path to the root directory of CTSM""" return _CTSM_ROOT + def path_to_cime(standalone_only=False): """Returns the path to cime, if it can be found @@ -41,7 +45,7 @@ def path_to_cime(standalone_only=False): that location. If standalone_only is False, then we fall back to checking where cime should be in a full CESM checkout. """ - cime_standalone_path = os.path.join(path_to_ctsm_root(), 'cime') + cime_standalone_path = os.path.join(path_to_ctsm_root(), "cime") if os.path.isdir(cime_standalone_path): return cime_standalone_path @@ -50,15 +54,20 @@ def path_to_cime(standalone_only=False): cesm_path = _path_to_cesm_root() if cesm_path is None: - raise RuntimeError("Cannot find cime within standalone CTSM checkout, " - "and we don't seem to be within a CESM checkout.") + raise RuntimeError( + "Cannot find cime within standalone CTSM checkout, " + "and we don't seem to be within a CESM checkout." + ) - cime_in_cesm_path = os.path.join(cesm_path, 'cime') + cime_in_cesm_path = os.path.join(cesm_path, "cime") if os.path.isdir(cime_in_cesm_path): return cime_in_cesm_path - raise RuntimeError("Cannot find cime within standalone CTSM checkout, " - "or within CESM checkout rooted at {}".format(cesm_path)) + raise RuntimeError( + "Cannot find cime within standalone CTSM checkout, " + "or within CESM checkout rooted at {}".format(cesm_path) + ) + def prepend_to_python_path(path): """Adds the given path to python's sys.path if it isn't already in the path @@ -70,6 +79,7 @@ def prepend_to_python_path(path): # Insert at location 1 rather than 0, because 0 is special sys.path.insert(1, path) + def add_cime_lib_to_path(standalone_only=False): """Adds the CIME python library to the python path, to allow importing modules from that library @@ -80,18 +90,18 @@ def add_cime_lib_to_path(standalone_only=False): path_to_cime """ cime_path = path_to_cime(standalone_only=standalone_only) - cime_lib_path = os.path.join(cime_path, - 'scripts', 'lib') + cime_lib_path = os.path.join(cime_path, "scripts", "lib") prepend_to_python_path(cime_lib_path) - cime_lib_path = os.path.join(cime_path, - 'scripts', 'Tools') + cime_lib_path = os.path.join(cime_path, "scripts", "Tools") prepend_to_python_path(cime_lib_path) return cime_path + # ======================================================================== # Private functions # ======================================================================== + def _path_to_cesm_root(): """Returns the path to the root directory of CESM, if we appear to be inside a CESM checkout. If we don't appear to be inside a CESM @@ -100,6 +110,6 @@ def _path_to_cesm_root(): ctsm_root = path_to_ctsm_root() for candidate_path in _CESM_CTSM_PATHS: if ctsm_root.endswith(candidate_path): - return os.path.normpath(ctsm_root[:-len(candidate_path)]) + return os.path.normpath(ctsm_root[: -len(candidate_path)]) return None diff --git a/python/ctsm/run_ctsm_py_tests.py b/python/ctsm/run_ctsm_py_tests.py index f0171a1940..91ce61a211 100644 --- a/python/ctsm/run_ctsm_py_tests.py +++ b/python/ctsm/run_ctsm_py_tests.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) + def main(description): """Main function called when run_tests is run from the command-line @@ -24,58 +25,60 @@ def main(description): if args.pattern is not None: pattern = args.pattern elif args.unit: - pattern = 'test_unit*.py' + pattern = "test_unit*.py" elif args.sys: - pattern = 'test_sys*.py' + pattern = "test_sys*.py" else: - pattern = 'test*.py' + pattern = "test*.py" # This setup_for_tests call is the main motivation for having this wrapper script to # run the tests rather than just using 'python -m unittest discover' unit_testing.setup_for_tests(enable_critical_logs=args.debug) mydir = os.path.dirname(os.path.abspath(__file__)) - testsuite = unittest.defaultTestLoader.discover( - start_dir=mydir, - pattern=pattern) + testsuite = unittest.defaultTestLoader.discover(start_dir=mydir, pattern=pattern) # NOTE(wjs, 2018-08-29) We may want to change the meaning of '--debug' # vs. '--verbose': I could imagine having --verbose set buffer=False, and --debug # additionally sets the logging level to much higher - e.g., debug level. - testrunner = unittest.TextTestRunner(buffer=(not args.debug), - verbosity=verbosity) + testrunner = unittest.TextTestRunner(buffer=(not args.debug), verbosity=verbosity) testrunner.run(testsuite) + def _commandline_args(description): - """Parse and return command-line arguments - """ + """Parse and return command-line arguments""" parser = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter) + description=description, formatter_class=argparse.RawTextHelpFormatter + ) output_level = parser.add_mutually_exclusive_group() - output_level.add_argument('-v', '--verbose', action='store_true', - help='Run tests with more verbosity') + output_level.add_argument( + "-v", "--verbose", action="store_true", help="Run tests with more verbosity" + ) - output_level.add_argument('-d', '--debug', action='store_true', - help='Run tests with even more verbosity') + output_level.add_argument( + "-d", "--debug", action="store_true", help="Run tests with even more verbosity" + ) test_subset = parser.add_mutually_exclusive_group() - test_subset.add_argument('-u', '--unit', action='store_true', - help='Only run unit tests') + test_subset.add_argument( + "-u", "--unit", action="store_true", help="Only run unit tests" + ) - test_subset.add_argument('-s', '--sys', action='store_true', - help='Only run system tests') + test_subset.add_argument( + "-s", "--sys", action="store_true", help="Only run system tests" + ) - test_subset.add_argument('-p', '--pattern', - help='File name pattern to match\n' - 'Default is test*.py') + test_subset.add_argument( + "-p", "--pattern", help="File name pattern to match\n" "Default is test*.py" + ) args = parser.parse_args() return args + def _get_verbosity_level(args): if args.debug or args.verbose: verbosity = 2 diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py index f45aa81927..da8e93a38f 100644 --- a/python/ctsm/run_sys_tests.py +++ b/python/ctsm/run_sys_tests.py @@ -11,7 +11,11 @@ from CIME.test_utils import get_tests_from_xml # pylint: disable=import-error from CIME.cs_status_creator import create_cs_status # pylint: disable=import-error -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) from ctsm.machine_utils import get_machine_name from ctsm.machine import create_machine, get_possibly_overridden_mach_value from ctsm.machine_defaults import MACHINE_DEFAULTS @@ -28,12 +32,13 @@ _NICE_LEVEL = 19 # Extra arguments for the cs.status.fails command -_CS_STATUS_FAILS_EXTRA_ARGS = '--fails-only --count-performance-fails' +_CS_STATUS_FAILS_EXTRA_ARGS = "--fails-only --count-performance-fails" # ======================================================================== # Public functions # ======================================================================== + def main(cime_path): """Main function called when run_sys_tests is run from the command-line @@ -46,48 +51,67 @@ def main(cime_path): setup_logging_pre_config() args = _commandline_args() process_logging_args(args) - logger.info('Running on machine: %s', args.machine_name) + logger.info("Running on machine: %s", args.machine_name) if args.job_launcher_nobatch: job_launcher_type = JOB_LAUNCHER_NOBATCH else: job_launcher_type = None - machine = create_machine(machine_name=args.machine_name, - job_launcher_type=job_launcher_type, - defaults=MACHINE_DEFAULTS, - account=args.account, - job_launcher_queue=args.job_launcher_queue, - job_launcher_walltime=args.job_launcher_walltime, - job_launcher_nice_level=_NICE_LEVEL, - job_launcher_extra_args=args.job_launcher_extra_args) + machine = create_machine( + machine_name=args.machine_name, + job_launcher_type=job_launcher_type, + defaults=MACHINE_DEFAULTS, + account=args.account, + job_launcher_queue=args.job_launcher_queue, + job_launcher_walltime=args.job_launcher_walltime, + job_launcher_nice_level=_NICE_LEVEL, + job_launcher_extra_args=args.job_launcher_extra_args, + ) logger.debug("Machine info: %s", machine) - run_sys_tests(machine=machine, cime_path=cime_path, - skip_testroot_creation=args.skip_testroot_creation, - skip_git_status=args.skip_git_status, - dry_run=args.dry_run, - suite_name=args.suite_name, testfile=args.testfile, testlist=args.testname, - suite_compilers=args.suite_compiler, - testid_base=args.testid_base, testroot_base=args.testroot_base, - rerun_existing_failures=args.rerun_existing_failures, - compare_name=args.compare, generate_name=args.generate, - baseline_root=args.baseline_root, - walltime=args.walltime, queue=args.queue, - retry=args.retry, - extra_create_test_args=args.extra_create_test_args) - -def run_sys_tests(machine, cime_path, - skip_testroot_creation=False, - skip_git_status=False, - dry_run=False, - suite_name=None, testfile=None, testlist=None, - suite_compilers=None, - testid_base=None, testroot_base=None, - rerun_existing_failures=False, - compare_name=None, generate_name=None, - baseline_root=None, - walltime=None, queue=None, - retry=None, - extra_create_test_args=''): + run_sys_tests( + machine=machine, + cime_path=cime_path, + skip_testroot_creation=args.skip_testroot_creation, + skip_git_status=args.skip_git_status, + dry_run=args.dry_run, + suite_name=args.suite_name, + testfile=args.testfile, + testlist=args.testname, + suite_compilers=args.suite_compiler, + testid_base=args.testid_base, + testroot_base=args.testroot_base, + rerun_existing_failures=args.rerun_existing_failures, + compare_name=args.compare, + generate_name=args.generate, + baseline_root=args.baseline_root, + walltime=args.walltime, + queue=args.queue, + retry=args.retry, + extra_create_test_args=args.extra_create_test_args, + ) + + +def run_sys_tests( + machine, + cime_path, + skip_testroot_creation=False, + skip_git_status=False, + dry_run=False, + suite_name=None, + testfile=None, + testlist=None, + suite_compilers=None, + testid_base=None, + testroot_base=None, + rerun_existing_failures=False, + compare_name=None, + generate_name=None, + baseline_root=None, + walltime=None, + queue=None, + retry=None, + extra_create_test_args="", +): """Implementation of run_sys_tests command Exactly one of suite_name, testfile or testlist should be provided @@ -127,11 +151,15 @@ def run_sys_tests(machine, cime_path, space-delimited string testlist: list of strings giving test names to run """ - num_provided_options = ((suite_name is not None) + - (testfile is not None) + - (testlist is not None and len(testlist) > 0)) + num_provided_options = ( + (suite_name is not None) + + (testfile is not None) + + (testlist is not None and len(testlist) > 0) + ) if num_provided_options != 1: - raise RuntimeError("Exactly one of suite_name, testfile or testlist must be provided") + raise RuntimeError( + "Exactly one of suite_name, testfile or testlist must be provided" + ) if testid_base is None: testid_base = _get_testid_base(machine.name) @@ -142,59 +170,72 @@ def run_sys_tests(machine, cime_path, _make_testroot(testroot, testid_base, dry_run) else: if not os.path.exists(testroot): - raise RuntimeError("testroot directory does NOT exist as expected when a rerun" + - " option is used: directory expected = "+testroot ) + raise RuntimeError( + "testroot directory does NOT exist as expected when a rerun" + + " option is used: directory expected = " + + testroot + ) print("Testroot: {}\n".format(testroot)) - retry_final = get_possibly_overridden_mach_value(machine, - varname='create_test_retry', - value=retry) + retry_final = get_possibly_overridden_mach_value( + machine, varname="create_test_retry", value=retry + ) if not skip_git_status: _record_git_status(testroot, retry_final, dry_run) - baseline_root_final = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value=baseline_root) - create_test_args = _get_create_test_args(compare_name=compare_name, - generate_name=generate_name, - baseline_root=baseline_root_final, - account=machine.account, - walltime=walltime, - queue=queue, - retry=retry_final, - rerun_existing_failures=rerun_existing_failures, - extra_create_test_args=extra_create_test_args) + baseline_root_final = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value=baseline_root + ) + create_test_args = _get_create_test_args( + compare_name=compare_name, + generate_name=generate_name, + baseline_root=baseline_root_final, + account=machine.account, + walltime=walltime, + queue=queue, + retry=retry_final, + rerun_existing_failures=rerun_existing_failures, + extra_create_test_args=extra_create_test_args, + ) if suite_name: if not dry_run: _make_cs_status_for_suite(testroot, testid_base) - _run_test_suite(cime_path=cime_path, - suite_name=suite_name, - suite_compilers=suite_compilers, - machine=machine, - testid_base=testid_base, testroot=testroot, - create_test_args=create_test_args, - dry_run=dry_run) + _run_test_suite( + cime_path=cime_path, + suite_name=suite_name, + suite_compilers=suite_compilers, + machine=machine, + testid_base=testid_base, + testroot=testroot, + create_test_args=create_test_args, + dry_run=dry_run, + ) else: if not dry_run: _make_cs_status_non_suite(testroot, testid_base) if testfile: - test_args = ['--testfile', os.path.abspath(testfile)] + test_args = ["--testfile", os.path.abspath(testfile)] elif testlist: test_args = testlist else: raise RuntimeError("None of suite_name, testfile or testlist were provided") - _run_create_test(cime_path=cime_path, - test_args=test_args, machine=machine, - testid=testid_base, testroot=testroot, - create_test_args=create_test_args, - dry_run=dry_run) + _run_create_test( + cime_path=cime_path, + test_args=test_args, + machine=machine, + testid=testid_base, + testroot=testroot, + create_test_args=create_test_args, + dry_run=dry_run, + ) + # ======================================================================== # Private functions # ======================================================================== + def _commandline_args(): - """Parse and return command-line arguments - """ + """Parse and return command-line arguments""" description = """ Driver for running CTSM system tests @@ -220,153 +261,215 @@ def _commandline_args(): """ parser = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter) + description=description, formatter_class=argparse.RawTextHelpFormatter + ) machine_name = get_machine_name() - default_machine = create_machine(machine_name, - defaults=MACHINE_DEFAULTS, - allow_missing_entries=True) + default_machine = create_machine( + machine_name, defaults=MACHINE_DEFAULTS, allow_missing_entries=True + ) tests_to_run = parser.add_mutually_exclusive_group(required=True) - tests_to_run.add_argument('-s', '--suite-name', - help='Name of test suite to run') + tests_to_run.add_argument("-s", "--suite-name", help="Name of test suite to run") - tests_to_run.add_argument('-f', '--testfile', - help='Path to file listing tests to run') + tests_to_run.add_argument( + "-f", "--testfile", help="Path to file listing tests to run" + ) - tests_to_run.add_argument('-t', '--testname', '--testnames', nargs='+', - help='One or more test names to run (space-delimited)') + tests_to_run.add_argument( + "-t", + "--testname", + "--testnames", + nargs="+", + help="One or more test names to run (space-delimited)", + ) compare = parser.add_mutually_exclusive_group(required=True) - compare.add_argument('-c', '--compare', metavar='COMPARE_NAME', - help='Baseline name (often tag) to compare against\n' - '(required unless --skip-compare is given)') + compare.add_argument( + "-c", + "--compare", + metavar="COMPARE_NAME", + help="Baseline name (often tag) to compare against\n" + "(required unless --skip-compare is given)", + ) - compare.add_argument('--skip-compare', action='store_true', - help='Do not compare against baselines') + compare.add_argument( + "--skip-compare", action="store_true", help="Do not compare against baselines" + ) generate = parser.add_mutually_exclusive_group(required=True) - generate.add_argument('-g', '--generate', metavar='GENERATE_NAME', - help='Baseline name (often tag) to generate\n' - '(required unless --skip-generate is given)') - - generate.add_argument('--skip-generate', action='store_true', - help='Do not generate baselines') - - parser.add_argument('--suite-compiler', '--suite-compilers', nargs='+', - help='Compiler(s) from the given test suite for which tests are run\n' - 'Only valid in conjunction with -s/--suite-name;\n' - 'if not specified, use all compilers defined for this suite and machine\n') - - parser.add_argument('--account', - help='Account number to use for job submission.\n' - 'This is needed on some machines; if not provided explicitly,\n' - 'the script will attempt to guess it using the same rules as in CIME.\n' - 'Default for this machine: {}'.format(default_machine.account)) - - parser.add_argument('--testid-base', - help='Base string used for the test id.\n' - 'Default is to auto-generate this with a date and time stamp.') - - parser.add_argument('--testroot-base', - help='Path in which testroot should be put.\n' - 'For supported machines, this can be left out;\n' - 'for non-supported machines, it must be provided.\n' - 'Default for this machine: {}'.format(default_machine.scratch_dir)) - - parser.add_argument('--rerun-existing-failures', action='store_true', - help='Rerun failed tests from the last PEND or FAIL state.\n' - 'This triggers the --use-existing option to create_test.\n' - 'To use this option, provide the same options to run_sys_tests\n' - 'as in the initial run, but also adding --testid-base\n' - 'corresponding to the base testid used initially.\n' - '(However, many of the arguments to create_test are ignored,\n' - 'so it is not important for all of the options to exactly match\n' - 'those in the initial run.)\n' - 'This option implies --skip-testroot-creation (that option does not\n' - 'need to be specified separately if using --rerun-existing-failures).') + generate.add_argument( + "-g", + "--generate", + metavar="GENERATE_NAME", + help="Baseline name (often tag) to generate\n" + "(required unless --skip-generate is given)", + ) + + generate.add_argument( + "--skip-generate", action="store_true", help="Do not generate baselines" + ) + + parser.add_argument( + "--suite-compiler", + "--suite-compilers", + nargs="+", + help="Compiler(s) from the given test suite for which tests are run\n" + "Only valid in conjunction with -s/--suite-name;\n" + "if not specified, use all compilers defined for this suite and machine\n", + ) + + parser.add_argument( + "--account", + help="Account number to use for job submission.\n" + "This is needed on some machines; if not provided explicitly,\n" + "the script will attempt to guess it using the same rules as in CIME.\n" + "Default for this machine: {}".format(default_machine.account), + ) + + parser.add_argument( + "--testid-base", + help="Base string used for the test id.\n" + "Default is to auto-generate this with a date and time stamp.", + ) + + parser.add_argument( + "--testroot-base", + help="Path in which testroot should be put.\n" + "For supported machines, this can be left out;\n" + "for non-supported machines, it must be provided.\n" + "Default for this machine: {}".format(default_machine.scratch_dir), + ) + + parser.add_argument( + "--rerun-existing-failures", + action="store_true", + help="Rerun failed tests from the last PEND or FAIL state.\n" + "This triggers the --use-existing option to create_test.\n" + "To use this option, provide the same options to run_sys_tests\n" + "as in the initial run, but also adding --testid-base\n" + "corresponding to the base testid used initially.\n" + "(However, many of the arguments to create_test are ignored,\n" + "so it is not important for all of the options to exactly match\n" + "those in the initial run.)\n" + "This option implies --skip-testroot-creation (that option does not\n" + "need to be specified separately if using --rerun-existing-failures).", + ) if default_machine.baseline_dir: - baseline_root_default_msg = 'Default for this machine: {}'.format( - default_machine.baseline_dir) + baseline_root_default_msg = "Default for this machine: {}".format( + default_machine.baseline_dir + ) else: baseline_root_default_msg = "Default for this machine: use cime's default" - parser.add_argument('--baseline-root', - help='Path in which baselines should be compared and generated.\n' + - baseline_root_default_msg) - - parser.add_argument('--walltime', - help='Walltime for each test.\n' - 'If running a test suite, you can generally leave this unset,\n' - 'because it is set in the file defining the test suite.\n' - 'For other uses, providing this helps decrease the time spent\n' - 'waiting in the queue.') - - parser.add_argument('--queue', - help='Queue to which tests are submitted.\n' - 'If not provided, uses machine default.') - - parser.add_argument('--retry', type=int, - help='Argument to create_test: Number of times to retry failed tests.\n' - 'Default for this machine: {}'.format( - default_machine.create_test_retry)) - - parser.add_argument('--extra-create-test-args', default='', - help='String giving extra arguments to pass to create_test\n' - '(To allow the argument parsing to accept this, enclose the string\n' - 'in quotes, with a leading space, as in " --my-arg foo".)') - - parser.add_argument('--job-launcher-nobatch', action='store_true', - help='Run create_test on the login node, even if this machine\n' - 'is set up to submit create_test to a compute node by default.') - - parser.add_argument('--job-launcher-queue', - help='Queue to which the create_test command is submitted.\n' - 'Only applies on machines for which we submit the create_test command\n' - 'rather than running it on the login node.\n' - 'Default for this machine: {}'.format( - default_machine.job_launcher.get_queue())) - - parser.add_argument('--job-launcher-walltime', - help='Walltime for the create_test command.\n' - 'Only applies on machines for which we submit the create_test command\n' - 'rather than running it on the login node.\n' - 'Default for this machine: {}'.format( - default_machine.job_launcher.get_walltime())) - - parser.add_argument('--job-launcher-extra-args', - help='Extra arguments for the command that launches the\n' - 'create_test command.\n' - '(To allow the argument parsing to accept this, enclose the string\n' - 'in quotes, with a leading space, as in " --my-arg foo".)\n' - 'Default for this machine: {}'.format( - default_machine.job_launcher.get_extra_args())) - - parser.add_argument('--skip-testroot-creation', action='store_true', - help='Do not create the directory that will hold the tests.\n' - 'This should be used if the desired testroot directory already exists.') - - parser.add_argument('--skip-git-status', action='store_true', - help='Skip printing git and manage_externals status,\n' - 'both to screen and to the SRCROOT_GIT_STATUS file in TESTROOT.\n' - 'This printing can often be helpful, but this option can be used to\n' - 'avoid extraneous output, to reduce the time needed to run this script,\n' - 'or if git or manage_externals are currently broken in your sandbox.\n') - - parser.add_argument('--dry-run', action='store_true', - help='Print what would happen, but do not run any commands.\n' - '(Generally should be run with --verbose.)\n') - - parser.add_argument('--machine-name', default=machine_name, - help='Name of machine for which create_test is run.\n' - 'This typically is not needed, but can be provided\n' - 'for the sake of testing this script.\n' - 'Defaults to current machine: {}'.format(machine_name)) + parser.add_argument( + "--baseline-root", + help="Path in which baselines should be compared and generated.\n" + + baseline_root_default_msg, + ) + + parser.add_argument( + "--walltime", + help="Walltime for each test.\n" + "If running a test suite, you can generally leave this unset,\n" + "because it is set in the file defining the test suite.\n" + "For other uses, providing this helps decrease the time spent\n" + "waiting in the queue.", + ) + + parser.add_argument( + "--queue", + help="Queue to which tests are submitted.\n" + "If not provided, uses machine default.", + ) + + parser.add_argument( + "--retry", + type=int, + help="Argument to create_test: Number of times to retry failed tests.\n" + "Default for this machine: {}".format(default_machine.create_test_retry), + ) + + parser.add_argument( + "--extra-create-test-args", + default="", + help="String giving extra arguments to pass to create_test\n" + "(To allow the argument parsing to accept this, enclose the string\n" + 'in quotes, with a leading space, as in " --my-arg foo".)', + ) + + parser.add_argument( + "--job-launcher-nobatch", + action="store_true", + help="Run create_test on the login node, even if this machine\n" + "is set up to submit create_test to a compute node by default.", + ) + + parser.add_argument( + "--job-launcher-queue", + help="Queue to which the create_test command is submitted.\n" + "Only applies on machines for which we submit the create_test command\n" + "rather than running it on the login node.\n" + "Default for this machine: {}".format(default_machine.job_launcher.get_queue()), + ) + + parser.add_argument( + "--job-launcher-walltime", + help="Walltime for the create_test command.\n" + "Only applies on machines for which we submit the create_test command\n" + "rather than running it on the login node.\n" + "Default for this machine: {}".format( + default_machine.job_launcher.get_walltime() + ), + ) + + parser.add_argument( + "--job-launcher-extra-args", + help="Extra arguments for the command that launches the\n" + "create_test command.\n" + "(To allow the argument parsing to accept this, enclose the string\n" + 'in quotes, with a leading space, as in " --my-arg foo".)\n' + "Default for this machine: {}".format( + default_machine.job_launcher.get_extra_args() + ), + ) + + parser.add_argument( + "--skip-testroot-creation", + action="store_true", + help="Do not create the directory that will hold the tests.\n" + "This should be used if the desired testroot directory already exists.", + ) + + parser.add_argument( + "--skip-git-status", + action="store_true", + help="Skip printing git and manage_externals status,\n" + "both to screen and to the SRCROOT_GIT_STATUS file in TESTROOT.\n" + "This printing can often be helpful, but this option can be used to\n" + "avoid extraneous output, to reduce the time needed to run this script,\n" + "or if git or manage_externals are currently broken in your sandbox.\n", + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Print what would happen, but do not run any commands.\n" + "(Generally should be run with --verbose.)\n", + ) + + parser.add_argument( + "--machine-name", + default=machine_name, + help="Name of machine for which create_test is run.\n" + "This typically is not needed, but can be provided\n" + "for the sake of testing this script.\n" + "Defaults to current machine: {}".format(machine_name), + ) add_logging_args(parser) @@ -376,32 +479,44 @@ def _commandline_args(): return args + def _check_arg_validity(args): if args.suite_compiler and not args.suite_name: - raise RuntimeError('--suite-compiler can only be specified if using --suite-name') + raise RuntimeError( + "--suite-compiler can only be specified if using --suite-name" + ) if args.rerun_existing_failures and not args.testid_base: - raise RuntimeError('With --rerun-existing-failures, must also specify --testid-base') + raise RuntimeError( + "With --rerun-existing-failures, must also specify --testid-base" + ) + def _get_testid_base(machine_name): """Returns a base testid based on the current date and time and the machine name""" now = datetime.now() now_str = now.strftime("%m%d-%H%M%S") machine_start = machine_name[0:2] - return '{}{}'.format(now_str, machine_start) + return "{}{}".format(now_str, machine_start) + def _get_testroot_base(machine): scratch_dir = machine.scratch_dir if scratch_dir is None: - raise RuntimeError('For a machine without a default specified for the scratch directory, ' - 'must specify --testroot-base') + raise RuntimeError( + "For a machine without a default specified for the scratch directory, " + "must specify --testroot-base" + ) return scratch_dir + def _get_testroot(testroot_base, testid_base): """Get the path to the test root, given a base test id""" return os.path.join(testroot_base, _get_testdir_name(testid_base)) + def _get_testdir_name(testid_base): - return 'tests_{}'.format(testid_base) + return "tests_{}".format(testid_base) + def _make_testroot(testroot, testid_base, dry_run): """Make the testroot directory at the given location, as well as a link in the current @@ -414,38 +529,41 @@ def _make_testroot(testroot, testid_base, dry_run): os.makedirs(testroot) make_link(testroot, _get_testdir_name(testid_base)) + def _record_git_status(testroot, retry, dry_run): """Record git status and related information to stdout and a file""" - output = '' + output = "" ctsm_root = path_to_ctsm_root() output += "create_test --retry: {}\n\n".format(retry) - current_hash = subprocess.check_output(['git', 'show', '--no-patch', - '--format=format:%h (%an, %ad) %s\n', 'HEAD'], - cwd=ctsm_root, - universal_newlines=True) + current_hash = subprocess.check_output( + ["git", "show", "--no-patch", "--format=format:%h (%an, %ad) %s\n", "HEAD"], + cwd=ctsm_root, + universal_newlines=True, + ) output += "Current hash: {}".format(current_hash) - git_status = subprocess.check_output(['git', '-c', 'color.ui=always', - 'status', '--short', '--branch'], - cwd=ctsm_root, - universal_newlines=True) + git_status = subprocess.check_output( + ["git", "-c", "color.ui=always", "status", "--short", "--branch"], + cwd=ctsm_root, + universal_newlines=True, + ) output += git_status - if git_status.count('\n') == 1: + if git_status.count("\n") == 1: # Only line in git status is the branch info output += "(clean sandbox)\n" - manic = os.path.join('manage_externals', 'checkout_externals') - manage_externals_status = subprocess.check_output([manic, '--status', '--verbose'], - cwd=ctsm_root, - universal_newlines=True) - output += 72*'-' + '\n' + 'manage_externals status:' + '\n' + manic = os.path.join("manage_externals", "checkout_externals") + manage_externals_status = subprocess.check_output( + [manic, "--status", "--verbose"], cwd=ctsm_root, universal_newlines=True + ) + output += 72 * "-" + "\n" + "manage_externals status:" + "\n" output += manage_externals_status - output += 72*'-' + '\n' + output += 72 * "-" + "\n" print(output) if not dry_run: - git_status_filepath = os.path.join(testroot, 'SRCROOT_GIT_STATUS') + git_status_filepath = os.path.join(testroot, "SRCROOT_GIT_STATUS") if os.path.exists(git_status_filepath): # If we're reusing an existing directory, it could happen that # SRCROOT_GIT_STATUS already exists. It's still helpful to record the current @@ -453,117 +571,163 @@ def _record_git_status(testroot, retry, dry_run): # make a new file with a date/time-stamp. now = datetime.now() now_str = now.strftime("%m%d-%H%M%S") - git_status_filepath = git_status_filepath + '_' + now_str - with open(git_status_filepath, 'w') as git_status_file: - git_status_file.write(' '.join(sys.argv) + '\n\n') + git_status_filepath = git_status_filepath + "_" + now_str + with open(git_status_filepath, "w") as git_status_file: + git_status_file.write(" ".join(sys.argv) + "\n\n") git_status_file.write("SRCROOT: {}\n".format(ctsm_root)) git_status_file.write(output) -def _get_create_test_args(compare_name, generate_name, baseline_root, - account, walltime, queue, retry, - rerun_existing_failures, - extra_create_test_args): + +def _get_create_test_args( + compare_name, + generate_name, + baseline_root, + account, + walltime, + queue, + retry, + rerun_existing_failures, + extra_create_test_args, +): args = [] if compare_name: - args.extend(['--compare', compare_name]) + args.extend(["--compare", compare_name]) if generate_name: - args.extend(['--generate', generate_name]) + args.extend(["--generate", generate_name]) if baseline_root: - args.extend(['--baseline-root', baseline_root]) + args.extend(["--baseline-root", baseline_root]) if account: - args.extend(['--project', account]) + args.extend(["--project", account]) if walltime: - args.extend(['--walltime', walltime]) + args.extend(["--walltime", walltime]) if queue: - args.extend(['--queue', queue]) - args.extend(['--retry', str(retry)]) + args.extend(["--queue", queue]) + args.extend(["--retry", str(retry)]) if rerun_existing_failures: # In addition to --use-existing, we also need --allow-baseline-overwrite in this # case; otherwise, create_test throws an error saying that the baseline # directories already exist. - args.extend(['--use-existing', '--allow-baseline-overwrite']) + args.extend(["--use-existing", "--allow-baseline-overwrite"]) args.extend(extra_create_test_args.split()) return args + def _make_cs_status_for_suite(testroot, testid_base): """Makes a cs.status file that can be run for the entire test suite""" - testid_pattern = testid_base + '_' + _NUM_COMPILER_CHARS*'?' + testid_pattern = testid_base + "_" + _NUM_COMPILER_CHARS * "?" # The basic cs.status just aggregates results from all of the individual create_tests - create_cs_status(test_root=testroot, - test_id=testid_pattern, - extra_args=_cs_status_xfail_arg(), - filename='cs.status') + create_cs_status( + test_root=testroot, + test_id=testid_pattern, + extra_args=_cs_status_xfail_arg(), + filename="cs.status", + ) # cs.status.fails additionally filters the results so that only failures are shown - create_cs_status(test_root=testroot, - test_id=testid_pattern, - extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()), - filename='cs.status.fails') + create_cs_status( + test_root=testroot, + test_id=testid_pattern, + extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + " " + _cs_status_xfail_arg()), + filename="cs.status.fails", + ) + def _make_cs_status_non_suite(testroot, testid_base): """Makes a cs.status file for a single run of create_test - not a whole test suite""" - create_cs_status(test_root=testroot, - test_id=testid_base, - extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()), - filename='cs.status.fails') + create_cs_status( + test_root=testroot, + test_id=testid_base, + extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + " " + _cs_status_xfail_arg()), + filename="cs.status.fails", + ) + def _cs_status_xfail_arg(): """Returns a string giving the argument to cs_status that will point to CTSM's expected fails xml file """ ctsm_root = path_to_ctsm_root() - xfail_path = os.path.join(ctsm_root, 'cime_config', 'testdefs', 'ExpectedTestFails.xml') + xfail_path = os.path.join( + ctsm_root, "cime_config", "testdefs", "ExpectedTestFails.xml" + ) return "--expected-fails-file {}".format(xfail_path) -def _run_test_suite(cime_path, suite_name, suite_compilers, - machine, testid_base, testroot, create_test_args, - dry_run): + +def _run_test_suite( + cime_path, + suite_name, + suite_compilers, + machine, + testid_base, + testroot, + create_test_args, + dry_run, +): if not suite_compilers: suite_compilers = _get_compilers_for_suite(suite_name, machine.name) for compiler in suite_compilers: - test_args = ['--xml-category', suite_name, - '--xml-machine', machine.name, - '--xml-compiler', compiler] - testid = testid_base + '_' + compiler[0:_NUM_COMPILER_CHARS] - _run_create_test(cime_path=cime_path, - test_args=test_args, machine=machine, - testid=testid, testroot=testroot, - create_test_args=create_test_args, - dry_run=dry_run) + test_args = [ + "--xml-category", + suite_name, + "--xml-machine", + machine.name, + "--xml-compiler", + compiler, + ] + testid = testid_base + "_" + compiler[0:_NUM_COMPILER_CHARS] + _run_create_test( + cime_path=cime_path, + test_args=test_args, + machine=machine, + testid=testid, + testroot=testroot, + create_test_args=create_test_args, + dry_run=dry_run, + ) + def _get_compilers_for_suite(suite_name, machine_name): - test_data = get_tests_from_xml( - xml_machine=machine_name, - xml_category=suite_name) + test_data = get_tests_from_xml(xml_machine=machine_name, xml_category=suite_name) if not test_data: - raise RuntimeError('No tests found for suite {} on machine {}'.format( - suite_name, machine_name)) - compilers = sorted({one_test['compiler'] for one_test in test_data}) + raise RuntimeError( + "No tests found for suite {} on machine {}".format(suite_name, machine_name) + ) + compilers = sorted({one_test["compiler"] for one_test in test_data}) logger.info("Running with compilers: %s", compilers) return compilers -def _run_create_test(cime_path, test_args, machine, testid, testroot, create_test_args, dry_run): - create_test_cmd = _build_create_test_cmd(cime_path=cime_path, - test_args=test_args, - testid=testid, - testroot=testroot, - create_test_args=create_test_args) - stdout_path = os.path.join(testroot, - 'STDOUT.{}'.format(testid)) - stderr_path = os.path.join(testroot, - 'STDERR.{}'.format(testid)) - machine.job_launcher.run_command(create_test_cmd, - stdout_path=stdout_path, - stderr_path=stderr_path, - dry_run=dry_run) + +def _run_create_test( + cime_path, test_args, machine, testid, testroot, create_test_args, dry_run +): + create_test_cmd = _build_create_test_cmd( + cime_path=cime_path, + test_args=test_args, + testid=testid, + testroot=testroot, + create_test_args=create_test_args, + ) + stdout_path = os.path.join(testroot, "STDOUT.{}".format(testid)) + stderr_path = os.path.join(testroot, "STDERR.{}".format(testid)) + machine.job_launcher.run_command( + create_test_cmd, + stdout_path=stdout_path, + stderr_path=stderr_path, + dry_run=dry_run, + ) + def _build_create_test_cmd(cime_path, test_args, testid, testroot, create_test_args): """Builds and returns the create_test command This is a list, where each element of the list is one argument """ - command = [os.path.join(cime_path, 'scripts', 'create_test'), - '--test-id', testid, - '--output-root', testroot] + command = [ + os.path.join(cime_path, "scripts", "create_test"), + "--test-id", + testid, + "--output-root", + testroot, + ] command.extend(test_args) command.extend(create_test_args) return command diff --git a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py index 53bb6dc07d..46b59f4e76 100755 --- a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py +++ b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py @@ -7,12 +7,16 @@ import tempfile import shutil import os -from ctsm.joblauncher.job_launcher_factory import create_job_launcher, JOB_LAUNCHER_NOBATCH +from ctsm.joblauncher.job_launcher_factory import ( + create_job_launcher, + JOB_LAUNCHER_NOBATCH, +) # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names # pylint: disable=invalid-name + class TestJobLauncherNoBatch(unittest.TestCase): """Tests of job_launcher_no_batch""" @@ -27,7 +31,7 @@ def assertFileContentsEqual(self, expected, filepath, msg=None): the string given by 'expected'. 'msg' gives an optional message to be printed if the assertion fails.""" - with open(filepath, 'r') as myfile: + with open(filepath, "r") as myfile: contents = myfile.read() self.assertEqual(expected, contents, msg=msg) @@ -35,21 +39,25 @@ def assertFileContentsEqual(self, expected, filepath, msg=None): def test_runCommand(self): """Test that the given command gets executed""" job_launcher = create_job_launcher(job_launcher_type=JOB_LAUNCHER_NOBATCH) - stdout = os.path.join(self._testdir, 'stdout') - job_launcher.run_command(command=['echo', 'hello', 'world'], - stdout_path=stdout, - stderr_path=os.path.join(self._testdir, 'stderr')) + stdout = os.path.join(self._testdir, "stdout") + job_launcher.run_command( + command=["echo", "hello", "world"], + stdout_path=stdout, + stderr_path=os.path.join(self._testdir, "stderr"), + ) job_launcher.wait_for_last_process_to_complete() self.assertTrue(os.path.isfile(stdout)) - self.assertFileContentsEqual('hello world\n', stdout) + self.assertFileContentsEqual("hello world\n", stdout) def test_runCommand_dryRun(self): """With dry_run, testdir should be empty""" job_launcher = create_job_launcher(job_launcher_type=JOB_LAUNCHER_NOBATCH) - job_launcher.run_command(command=['echo', 'hello', 'world'], - stdout_path=os.path.join(self._testdir, 'stdout'), - stderr_path=os.path.join(self._testdir, 'stderr'), - dry_run=True) + job_launcher.run_command( + command=["echo", "hello", "world"], + stdout_path=os.path.join(self._testdir, "stdout"), + stderr_path=os.path.join(self._testdir, "stderr"), + dry_run=True, + ) # There shouldn't be a "last process", but in case there is, wait for it to # complete so we can be confident that the test isn't passing simply because the # process hasn't completed yet. (This relies on there being logic in diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index 3c4117fd45..b2f62d8efa 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -21,6 +21,7 @@ # to make readable unit test names # pylint: disable=invalid-name + class TestSysBuildCtsm(unittest.TestCase): """System tests for lilac_build_ctsm""" @@ -39,23 +40,25 @@ def test_buildSetup_userDefinedMachine_minimalInfo(self): This version specifies a minimal amount of information """ - build_dir = os.path.join(self._tempdir, 'ctsm_build') - build_ctsm(cime_path=_CIME_PATH, - build_dir=build_dir, - compiler='gnu', - no_build=True, - os_type='linux', - netcdf_path='/path/to/netcdf', - esmf_mkfile_path='/path/to/esmf/lib/esmf.mk', - max_mpitasks_per_node=16, - gmake='gmake', - gmake_j=8, - no_pnetcdf=True) + build_dir = os.path.join(self._tempdir, "ctsm_build") + build_ctsm( + cime_path=_CIME_PATH, + build_dir=build_dir, + compiler="gnu", + no_build=True, + os_type="linux", + netcdf_path="/path/to/netcdf", + esmf_mkfile_path="/path/to/esmf/lib/esmf.mk", + max_mpitasks_per_node=16, + gmake="gmake", + gmake_j=8, + no_pnetcdf=True, + ) # the critical piece of this test is that the above command doesn't generate any # errors; however we also do some assertions below # ensure that inputdata directory was created - inputdata = os.path.join(build_dir, 'inputdata') + inputdata = os.path.join(build_dir, "inputdata") self.assertTrue(os.path.isdir(inputdata)) def test_buildSetup_userDefinedMachine_allInfo(self): @@ -67,34 +70,37 @@ def test_buildSetup_userDefinedMachine_allInfo(self): This version specifies all possible information """ - build_dir = os.path.join(self._tempdir, 'ctsm_build') - inputdata_path = os.path.realpath(os.path.join(self._tempdir, 'my_inputdata')) + build_dir = os.path.join(self._tempdir, "ctsm_build") + inputdata_path = os.path.realpath(os.path.join(self._tempdir, "my_inputdata")) os.makedirs(inputdata_path) - build_ctsm(cime_path=_CIME_PATH, - build_dir=build_dir, - compiler='gnu', - no_build=True, - os_type='linux', - netcdf_path='/path/to/netcdf', - esmf_mkfile_path='/path/to/esmf/lib/esmf.mk', - max_mpitasks_per_node=16, - gmake='gmake', - gmake_j=8, - pnetcdf_path='/path/to/pnetcdf', - pio_filesystem_hints='gpfs', - gptl_nano_timers=True, - extra_fflags='-foo', - extra_cflags='-bar', - build_debug=True, - build_with_openmp=True, - inputdata_path=os.path.join(self._tempdir, 'my_inputdata')) + build_ctsm( + cime_path=_CIME_PATH, + build_dir=build_dir, + compiler="gnu", + no_build=True, + os_type="linux", + netcdf_path="/path/to/netcdf", + esmf_mkfile_path="/path/to/esmf/lib/esmf.mk", + max_mpitasks_per_node=16, + gmake="gmake", + gmake_j=8, + pnetcdf_path="/path/to/pnetcdf", + pio_filesystem_hints="gpfs", + gptl_nano_timers=True, + extra_fflags="-foo", + extra_cflags="-bar", + build_debug=True, + build_with_openmp=True, + inputdata_path=os.path.join(self._tempdir, "my_inputdata"), + ) # the critical piece of this test is that the above command doesn't generate any # errors; however we also do some assertions below # ensure that inputdata directory is NOT created - inputdata = os.path.join(build_dir, 'inputdata') + inputdata = os.path.join(build_dir, "inputdata") self.assertFalse(os.path.exists(inputdata)) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_lilac_build_ctsm.py b/python/ctsm/test/test_unit_lilac_build_ctsm.py index 96f79f3765..12719697e2 100755 --- a/python/ctsm/test/test_unit_lilac_build_ctsm.py +++ b/python/ctsm/test/test_unit_lilac_build_ctsm.py @@ -16,6 +16,7 @@ # pylint: disable=line-too-long + class TestBuildCtsm(unittest.TestCase): """Tests of lilac_build_ctsm""" @@ -26,9 +27,9 @@ class TestBuildCtsm(unittest.TestCase): def test_commandlineArgs_rebuild_valid(self): """Test _commandline_args with --rebuild, with a valid argument list (no disallowed args)""" # pylint: disable=no-self-use - _ = _commandline_args(args_to_parse=['build/directory', '--rebuild']) + _ = _commandline_args(args_to_parse=["build/directory", "--rebuild"]) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_rebuild_invalid1(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is invalid with this option @@ -36,12 +37,12 @@ def test_commandlineArgs_rebuild_invalid1(self, mock_stderr): """ expected_re = r"--compiler cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--rebuild', - '--compiler', 'intel']) + _ = _commandline_args( + args_to_parse=["build/directory", "--rebuild", "--compiler", "intel"] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_rebuild_invalid2(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is invalid with this option @@ -49,12 +50,12 @@ def test_commandlineArgs_rebuild_invalid2(self, mock_stderr): """ expected_re = r"--os cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--rebuild', - '--os', 'linux']) + _ = _commandline_args( + args_to_parse=["build/directory", "--rebuild", "--os", "linux"] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_rebuild_invalid3(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is invalid with this option @@ -62,12 +63,17 @@ def test_commandlineArgs_rebuild_invalid3(self, mock_stderr): """ expected_re = r"--netcdf-path cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--rebuild', - '--netcdf-path', '/path/to/netcdf']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--rebuild", + "--netcdf-path", + "/path/to/netcdf", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_rebuild_invalid4(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is invalid with this option @@ -76,12 +82,12 @@ def test_commandlineArgs_rebuild_invalid4(self, mock_stderr): """ expected_re = r"--no-build cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--rebuild', - '--no-build']) + _ = _commandline_args( + args_to_parse=["build/directory", "--rebuild", "--no-build"] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_rebuild_invalid5(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is invalid with this option @@ -90,9 +96,9 @@ def test_commandlineArgs_rebuild_invalid5(self, mock_stderr): """ expected_re = r"--gmake cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--rebuild', - '--gmake', 'mymake']) + _ = _commandline_args( + args_to_parse=["build/directory", "--rebuild", "--gmake", "mymake"] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) def test_commandlineArgs_noRebuild_valid(self): @@ -101,15 +107,24 @@ def test_commandlineArgs_noRebuild_valid(self): (all required things present) """ # pylint: disable=no-self-use - _ = _commandline_args(args_to_parse=['build/directory', - '--os', 'linux', - '--compiler', 'intel', - '--netcdf-path', '/path/to/netcdf', - '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', - '--max-mpitasks-per-node', '16', - '--no-pnetcdf']) - - @patch('sys.stderr', new_callable=StringIO) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--os", + "linux", + "--compiler", + "intel", + "--netcdf-path", + "/path/to/netcdf", + "--esmf-mkfile-path", + "/path/to/esmf/lib/esmf.mk", + "--max-mpitasks-per-node", + "16", + "--no-pnetcdf", + ] + ) + + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_noRebuild_invalid1(self, mock_stderr): """Test _commandline_args without --rebuild or --machine, with a missing required argument @@ -117,56 +132,93 @@ def test_commandlineArgs_noRebuild_invalid1(self, mock_stderr): """ expected_re = r"--compiler must be provided if --rebuild is not set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--os', 'linux', - '--netcdf-path', '/path/to/netcdf', - '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', - '--max-mpitasks-per-node', '16', - '--no-pnetcdf']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--os", + "linux", + "--netcdf-path", + "/path/to/netcdf", + "--esmf-mkfile-path", + "/path/to/esmf/lib/esmf.mk", + "--max-mpitasks-per-node", + "16", + "--no-pnetcdf", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_noRebuild_invalid2(self, mock_stderr): """Test _commandline_args without --rebuild or --machine, with a missing required argument This tests an argument in the new-machine-required list """ - expected_re = r"--os must be provided if neither --machine nor --rebuild are set" + expected_re = ( + r"--os must be provided if neither --machine nor --rebuild are set" + ) with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--compiler', 'intel', - '--netcdf-path', '/path/to/netcdf', - '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', - '--max-mpitasks-per-node', '16', - '--no-pnetcdf']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--compiler", + "intel", + "--netcdf-path", + "/path/to/netcdf", + "--esmf-mkfile-path", + "/path/to/esmf/lib/esmf.mk", + "--max-mpitasks-per-node", + "16", + "--no-pnetcdf", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_noRebuild_invalidNeedToDictatePnetcdf(self, mock_stderr): """Test _commandline_args without --rebuild or --machine: invalid b/c nothing specified for pnetcdf""" expected_re = r"For a user-defined machine, need to specify either --no-pnetcdf or --pnetcdf-path" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--os', 'linux', - '--compiler', 'intel', - '--netcdf-path', '/path/to/netcdf', - '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', - '--max-mpitasks-per-node', '16']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--os", + "linux", + "--compiler", + "intel", + "--netcdf-path", + "/path/to/netcdf", + "--esmf-mkfile-path", + "/path/to/esmf/lib/esmf.mk", + "--max-mpitasks-per-node", + "16", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_noRebuild_invalidConflictingPnetcdf(self, mock_stderr): """Test _commandline_args without --rebuild or --machine: invalid b/c of conflicting specifications for pnetcdf""" expected_re = r"--no-pnetcdf cannot be given if you set --pnetcdf-path" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--os', 'linux', - '--compiler', 'intel', - '--netcdf-path', '/path/to/netcdf', - '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', - '--max-mpitasks-per-node', '16', - '--no-pnetcdf', - '--pnetcdf-path', '/path/to/pnetcdf']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--os", + "linux", + "--compiler", + "intel", + "--netcdf-path", + "/path/to/netcdf", + "--esmf-mkfile-path", + "/path/to/esmf/lib/esmf.mk", + "--max-mpitasks-per-node", + "16", + "--no-pnetcdf", + "--pnetcdf-path", + "/path/to/pnetcdf", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) def test_commandlineArgs_machine_valid(self): @@ -175,21 +227,27 @@ def test_commandlineArgs_machine_valid(self): (all required things present) """ # pylint: disable=no-self-use - _ = _commandline_args(args_to_parse=['build/directory', - '--machine', 'mymachine', - '--compiler', 'intel']) - - @patch('sys.stderr', new_callable=StringIO) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--machine", + "mymachine", + "--compiler", + "intel", + ] + ) + + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_machine_missingRequired(self, mock_stderr): - """Test _commandline_args with --machine, with a missing required argument - """ + """Test _commandline_args with --machine, with a missing required argument""" expected_re = r"--compiler must be provided if --rebuild is not set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--machine', 'mymachine']) + _ = _commandline_args( + args_to_parse=["build/directory", "--machine", "mymachine"] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_machine_illegalArg1(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is illegal with this option @@ -197,13 +255,20 @@ def test_commandlineArgs_machine_illegalArg1(self, mock_stderr): """ expected_re = r"--os cannot be provided if --machine is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--machine', 'mymachine', - '--compiler', 'intel', - '--os', 'linux']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--machine", + "mymachine", + "--compiler", + "intel", + "--os", + "linux", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) - @patch('sys.stderr', new_callable=StringIO) + @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_machine_illegalArg2(self, mock_stderr): """Test _commandline_args with --rebuild, with an argument that is illegal with this option @@ -211,10 +276,17 @@ def test_commandlineArgs_machine_illegalArg2(self, mock_stderr): """ expected_re = r"--gmake cannot be provided if --machine is set" with self.assertRaises(SystemExit): - _ = _commandline_args(args_to_parse=['build/directory', - '--machine', 'mymachine', - '--compiler', 'intel', - '--gmake', 'mymake']) + _ = _commandline_args( + args_to_parse=[ + "build/directory", + "--machine", + "mymachine", + "--compiler", + "intel", + "--gmake", + "mymake", + ] + ) self.assertRegex(mock_stderr.getvalue(), expected_re) # ------------------------------------------------------------------------ @@ -223,14 +295,15 @@ def test_commandlineArgs_machine_illegalArg2(self, mock_stderr): def test_checkAndTransformOs_valid(self): """Test _check_and_transform_os with valid input""" - os = _check_and_transform_os('linux') - self.assertEqual(os, 'LINUX') + os = _check_and_transform_os("linux") + self.assertEqual(os, "LINUX") def test_checkAndTransformOs_invalid(self): """Test _check_and_transform_os with invalid input""" with self.assertRaises(ValueError): - _ = _check_and_transform_os('bad_os') + _ = _check_and_transform_os("bad_os") + -if __name__ == '__main__': +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py b/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py index e6b602b3d7..ec21204f8e 100755 --- a/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py +++ b/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py @@ -12,43 +12,48 @@ # to make readable unit test names # pylint: disable=invalid-name + class TestMakeRuntimeInputs(unittest.TestCase): """Tests of lilac_make_runtime_inputs""" def test_buildnmlOpts_bgc(self): """Test determine_buildnml_opts with bgc_mode='bgc'""" - bldnml_opts = determine_bldnml_opts(bgc_mode='bgc', crop='off', vichydro='off') - self.assertRegex(bldnml_opts, r'^ *-bgc bgc *$') + bldnml_opts = determine_bldnml_opts(bgc_mode="bgc", crop="off", vichydro="off") + self.assertRegex(bldnml_opts, r"^ *-bgc bgc *$") def test_buildnmlOpts_fates(self): """Test determine_buildnml_opts with bgc_mode='fates'""" - bldnml_opts = determine_bldnml_opts(bgc_mode='fates', crop='off', vichydro='off') - self.assertRegex(bldnml_opts, r'^ *-bgc fates +-no-megan *$') + bldnml_opts = determine_bldnml_opts( + bgc_mode="fates", crop="off", vichydro="off" + ) + self.assertRegex(bldnml_opts, r"^ *-bgc fates +-no-megan *$") def test_buildnmlOpts_bgcCrop(self): """Test determine_buildnml_opts with bgc_mode='bgc' and crop on""" - bldnml_opts = determine_bldnml_opts(bgc_mode='bgc', crop='on', vichydro='off') - self.assertRegex(bldnml_opts, r'^ *-bgc bgc +-crop *$') + bldnml_opts = determine_bldnml_opts(bgc_mode="bgc", crop="on", vichydro="off") + self.assertRegex(bldnml_opts, r"^ *-bgc bgc +-crop *$") def test_buildnmlOpts_spCrop_fails(self): """Test determine_buildnml_opts with bgc_mode='sp' and crop on: should fail""" with self.assertRaisesRegex( - SystemExit, - "setting crop to 'on' is only compatible with bgc_mode"): - _ = determine_bldnml_opts(bgc_mode='sp', crop='on', vichydro='off') + SystemExit, "setting crop to 'on' is only compatible with bgc_mode" + ): + _ = determine_bldnml_opts(bgc_mode="sp", crop="on", vichydro="off") def test_buildnmlOpts_spVic(self): """Test determine_buildnml_opts with bgc_mode='sp' and vic on""" - bldnml_opts = determine_bldnml_opts(bgc_mode='sp', crop='off', vichydro='on') - self.assertRegex(bldnml_opts, r'^ *-bgc sp +-vichydro *$') + bldnml_opts = determine_bldnml_opts(bgc_mode="sp", crop="off", vichydro="on") + self.assertRegex(bldnml_opts, r"^ *-bgc sp +-vichydro *$") def test_buildnmlOpts_bgcVic(self): """Test determine_buildnml_opts with bgc_mode='bgc' and vic on: should fail""" with self.assertRaisesRegex( - SystemExit, - "setting vichydro to 'on' is only compatible with bgc_mode of 'sp'"): - _ = determine_bldnml_opts(bgc_mode='bgc', crop='off', vichydro='on') + SystemExit, + "setting vichydro to 'on' is only compatible with bgc_mode of 'sp'", + ): + _ = determine_bldnml_opts(bgc_mode="bgc", crop="off", vichydro="on") + -if __name__ == '__main__': +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_machine.py b/python/ctsm/test/test_unit_machine.py index 6a2f7ac172..9051fa4dd8 100755 --- a/python/ctsm/test/test_unit_machine.py +++ b/python/ctsm/test/test_unit_machine.py @@ -6,7 +6,7 @@ import unittest import os -from ctsm import add_cime_to_path # pylint: disable=unused-import +from ctsm import add_cime_to_path # pylint: disable=unused-import from ctsm import unit_testing from ctsm.machine import create_machine, get_possibly_overridden_mach_value @@ -14,17 +14,22 @@ from ctsm.machine_defaults import MACHINE_DEFAULTS, MachineDefaults, QsubDefaults from ctsm.joblauncher.job_launcher_no_batch import JobLauncherNoBatch from ctsm.joblauncher.job_launcher_qsub import JobLauncherQsub -from ctsm.joblauncher.job_launcher_factory import JOB_LAUNCHER_QSUB, JOB_LAUNCHER_NOBATCH +from ctsm.joblauncher.job_launcher_factory import ( + JOB_LAUNCHER_QSUB, + JOB_LAUNCHER_NOBATCH, +) # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names # pylint: disable=invalid-name + class TestCreateMachine(unittest.TestCase): """Tests of create_machine""" - def assertMachineInfo(self, machine, name, scratch_dir, baseline_dir, account, - create_test_retry=0): + def assertMachineInfo( + self, machine, name, scratch_dir, baseline_dir, account, create_test_retry=0 + ): """Asserts that the basic machine info is as expected. This does NOT dive down into the job launcher""" @@ -43,7 +48,9 @@ def assertNoBatchInfo(self, machine, nice_level=None): nice_level = 0 self.assertEqual(launcher.get_nice_level(), nice_level) - def assertQsubInfo(self, machine, queue, walltime, account, required_args, extra_args): + def assertQsubInfo( + self, machine, queue, walltime, account, required_args, extra_args + ): """Asserts that the machine's launcher is of type JobLauncherQsub, and has values as expected""" launcher = machine.job_launcher @@ -59,111 +66,135 @@ def create_defaults(default_job_launcher=JOB_LAUNCHER_QSUB): """Creates test-specific defaults so we don't tie the tests to changes in the real defaults""" defaults = { - 'cheyenne': MachineDefaults( + "cheyenne": MachineDefaults( job_launcher_type=default_job_launcher, - scratch_dir=os.path.join(os.path.sep, 'glade', 'scratch', get_user()), - baseline_dir=os.path.join(os.path.sep, 'my', 'baselines'), + scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), + baseline_dir=os.path.join(os.path.sep, "my", "baselines"), account_required=True, create_test_retry=2, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='regular', - walltime='06:00:00', - extra_args='', - required_args= - '-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login') - }) - } + queue="regular", + walltime="06:00:00", + extra_args="", + required_args="-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login", + ) + }, + ) + } return defaults def test_unknownMachine_defaults(self): """Tests a machine not in the defaults structure, with no overriding arguments""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - account='a123') - self.assertMachineInfo(machine=machine, - name='unknown_test_machine', - scratch_dir=None, - baseline_dir=None, - account='a123') + machine = create_machine( + "unknown_test_machine", MACHINE_DEFAULTS, account="a123" + ) + self.assertMachineInfo( + machine=machine, + name="unknown_test_machine", + scratch_dir=None, + baseline_dir=None, + account="a123", + ) self.assertNoBatchInfo(machine) def test_noBatchMachine_niceLevel(self): """Tests a no-batch machine where the nice level is explicit""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - job_launcher_type=JOB_LAUNCHER_NOBATCH, - scratch_dir='/path/to/scratch', - account='a123', - job_launcher_nice_level=13) - self.assertMachineInfo(machine=machine, - name='unknown_test_machine', - scratch_dir='/path/to/scratch', - baseline_dir=None, - account='a123') + machine = create_machine( + "unknown_test_machine", + MACHINE_DEFAULTS, + job_launcher_type=JOB_LAUNCHER_NOBATCH, + scratch_dir="/path/to/scratch", + account="a123", + job_launcher_nice_level=13, + ) + self.assertMachineInfo( + machine=machine, + name="unknown_test_machine", + scratch_dir="/path/to/scratch", + baseline_dir=None, + account="a123", + ) self.assertNoBatchInfo(machine, nice_level=13) def test_unknownMachine_argsExplicit(self): """Tests a machine not in the defaults structure, with explicit arguments""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - scratch_dir='/path/to/scratch', - job_launcher_type=JOB_LAUNCHER_QSUB, - account='a123', - job_launcher_queue='my_queue', - job_launcher_walltime='1:23:45', - job_launcher_extra_args='--some args') - self.assertMachineInfo(machine=machine, - name='unknown_test_machine', - scratch_dir='/path/to/scratch', - baseline_dir=None, - account='a123') - self.assertQsubInfo(machine=machine, - queue='my_queue', - walltime='1:23:45', - account='a123', - required_args='', - extra_args='--some args') + machine = create_machine( + "unknown_test_machine", + MACHINE_DEFAULTS, + scratch_dir="/path/to/scratch", + job_launcher_type=JOB_LAUNCHER_QSUB, + account="a123", + job_launcher_queue="my_queue", + job_launcher_walltime="1:23:45", + job_launcher_extra_args="--some args", + ) + self.assertMachineInfo( + machine=machine, + name="unknown_test_machine", + scratch_dir="/path/to/scratch", + baseline_dir=None, + account="a123", + ) + self.assertQsubInfo( + machine=machine, + queue="my_queue", + walltime="1:23:45", + account="a123", + required_args="", + extra_args="--some args", + ) def test_knownMachine_defaults(self): """Tests a machine known in the defaults structure, with no overriding arguments""" defaults = self.create_defaults() - machine = create_machine('cheyenne', defaults, account='a123') - self.assertMachineInfo(machine=machine, - name='cheyenne', - scratch_dir=os.path.join(os.path.sep, - 'glade', - 'scratch', - get_user()), - baseline_dir=os.path.join(os.path.sep, 'my', 'baselines'), - account='a123', - create_test_retry=2) - self.assertQsubInfo(machine=machine, - queue='regular', - walltime='06:00:00', - account='a123', - required_args='-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login', - extra_args='') + machine = create_machine("cheyenne", defaults, account="a123") + self.assertMachineInfo( + machine=machine, + name="cheyenne", + scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), + baseline_dir=os.path.join(os.path.sep, "my", "baselines"), + account="a123", + create_test_retry=2, + ) + self.assertQsubInfo( + machine=machine, + queue="regular", + walltime="06:00:00", + account="a123", + required_args="-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login", + extra_args="", + ) def test_knownMachine_argsExplicit(self): """Tests a machine known in the defaults structure, with explicit arguments""" defaults = self.create_defaults(default_job_launcher=JOB_LAUNCHER_NOBATCH) - machine = create_machine('cheyenne', defaults, - job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir='/custom/path/to/scratch', - account='a123', - job_launcher_queue='custom_queue', - job_launcher_walltime='9:87:65', - job_launcher_extra_args='--custom args') - self.assertMachineInfo(machine=machine, - name='cheyenne', - scratch_dir='/custom/path/to/scratch', - baseline_dir=os.path.join(os.path.sep, 'my', 'baselines'), - account='a123', - create_test_retry=2) - self.assertQsubInfo(machine=machine, - queue='custom_queue', - walltime='9:87:65', - account='a123', - required_args='-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login', - extra_args='--custom args') + machine = create_machine( + "cheyenne", + defaults, + job_launcher_type=JOB_LAUNCHER_QSUB, + scratch_dir="/custom/path/to/scratch", + account="a123", + job_launcher_queue="custom_queue", + job_launcher_walltime="9:87:65", + job_launcher_extra_args="--custom args", + ) + self.assertMachineInfo( + machine=machine, + name="cheyenne", + scratch_dir="/custom/path/to/scratch", + baseline_dir=os.path.join(os.path.sep, "my", "baselines"), + account="a123", + create_test_retry=2, + ) + self.assertQsubInfo( + machine=machine, + queue="custom_queue", + walltime="9:87:65", + account="a123", + required_args="-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login", + extra_args="--custom args", + ) # ------------------------------------------------------------------------ # Tests of get_possibly_overridden_mach_value @@ -172,31 +203,33 @@ def test_knownMachine_argsExplicit(self): def test_baselineDir_overridden(self): """Tests get_possibly_overridden_mach_value when baseline_dir is provided""" defaults = self.create_defaults() - machine = create_machine('cheyenne', defaults, account='a123') - baseline_dir = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value='mypath') - self.assertEqual(baseline_dir, 'mypath') + machine = create_machine("cheyenne", defaults, account="a123") + baseline_dir = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value="mypath" + ) + self.assertEqual(baseline_dir, "mypath") def test_baselineDir_default(self): """Tests get_possibly_overridden_mach_value when baseline_dir is not provided""" defaults = self.create_defaults() - machine = create_machine('cheyenne', defaults, account='a123') - baseline_dir = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value=None) - self.assertEqual(baseline_dir, os.path.join(os.path.sep, 'my', 'baselines')) + machine = create_machine("cheyenne", defaults, account="a123") + baseline_dir = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value=None + ) + self.assertEqual(baseline_dir, os.path.join(os.path.sep, "my", "baselines")) def test_baselineDir_noDefault(self): """Tests get_possibly_overridden_mach_value when baseline_dir is not provided and there is no default""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - account='a123') - baseline_dir = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value=None) + machine = create_machine( + "unknown_test_machine", MACHINE_DEFAULTS, account="a123" + ) + baseline_dir = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value=None + ) self.assertIsNone(baseline_dir) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_path_utils.py b/python/ctsm/test/test_unit_path_utils.py index 9fc996aa2c..566c458acc 100755 --- a/python/ctsm/test/test_unit_path_utils.py +++ b/python/ctsm/test/test_unit_path_utils.py @@ -17,6 +17,7 @@ # to make readable unit test names # pylint: disable=invalid-name + class TestPathUtils(unittest.TestCase): """Tests of path_utils""" @@ -31,14 +32,14 @@ def _ctsm_path_in_cesm(self): directory structure, where self._testdir is the root of the cesm checkout """ - return os.path.join(self._testdir, 'components', 'clm') + return os.path.join(self._testdir, "components", "clm") def _cime_path_in_cesm(self): """Returns the path to a cime directory nested inside a typical cesm directory structure, where self._testdir is the root of the cesm checkout """ - return os.path.join(self._testdir, 'cime') + return os.path.join(self._testdir, "cime") def _make_cesm_dirs(self): """Makes a directory structure for a typical CESM layout, where @@ -58,23 +59,21 @@ def test_pathToCime_standaloneOnlyWithCime(self): """Test path_to_cime with standalone_only, where cime is in the location it should be with a standalone checkout """ - ctsm_path = os.path.join(self._testdir, 'ctsm') - actual_path_to_cime = os.path.join(ctsm_path, 'cime') + ctsm_path = os.path.join(self._testdir, "ctsm") + actual_path_to_cime = os.path.join(ctsm_path, "cime") os.makedirs(actual_path_to_cime) - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): path_to_cime = path_utils.path_to_cime(standalone_only=True) self.assertEqual(path_to_cime, actual_path_to_cime) def test_pathToCime_standaloneOnlyWithoutCime(self): """Test path_to_cime with standalone_only, where cime is missing""" - ctsm_path = os.path.join(self._testdir, 'ctsm') + ctsm_path = os.path.join(self._testdir, "ctsm") os.makedirs(ctsm_path) - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): with six.assertRaisesRegex(self, RuntimeError, "Cannot find cime"): _ = path_utils.path_to_cime(standalone_only=True) @@ -86,8 +85,7 @@ def test_pathToCime_standaloneOnlyWithCimeInCesm(self): """ ctsm_path, _ = self._make_cesm_dirs() - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): with six.assertRaisesRegex(self, RuntimeError, "Cannot find cime"): _ = path_utils.path_to_cime(standalone_only=True) @@ -97,8 +95,7 @@ def test_pathToCime_cimeInCesm(self): """ ctsm_path, actual_path_to_cime = self._make_cesm_dirs() - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): path_to_cime = path_utils.path_to_cime() self.assertEqual(path_to_cime, actual_path_to_cime) @@ -107,14 +104,16 @@ def test_pathToCime_notInCesmCheckout(self): """Test path_to_cime, where cime is not in the standalone directory, and we don't appear to be in a CESM checkout """ - ctsm_path = os.path.join(self._testdir, 'components', 'foo') + ctsm_path = os.path.join(self._testdir, "components", "foo") os.makedirs(ctsm_path) os.makedirs(self._cime_path_in_cesm()) - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): - with six.assertRaisesRegex(self, RuntimeError, - "Cannot find cime.*don't seem to be within a CESM checkout"): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): + with six.assertRaisesRegex( + self, + RuntimeError, + "Cannot find cime.*don't seem to be within a CESM checkout", + ): _ = path_utils.path_to_cime() def test_pathToCime_noCimeInCesm(self): @@ -123,10 +122,10 @@ def test_pathToCime_noCimeInCesm(self): ctsm_path = self._ctsm_path_in_cesm() os.makedirs(ctsm_path) - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): - with six.assertRaisesRegex(self, RuntimeError, - "Cannot find cime.*or within CESM checkout"): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): + with six.assertRaisesRegex( + self, RuntimeError, "Cannot find cime.*or within CESM checkout" + ): _ = path_utils.path_to_cime() def test_pathToCime_cimeInStandaloneAndCesm(self): @@ -135,15 +134,15 @@ def test_pathToCime_cimeInStandaloneAndCesm(self): give us the cime in the standalone checkout. """ ctsm_path, _ = self._make_cesm_dirs() - actual_path_to_cime = os.path.join(ctsm_path, 'cime') + actual_path_to_cime = os.path.join(ctsm_path, "cime") os.makedirs(actual_path_to_cime) - with mock.patch('ctsm.path_utils.path_to_ctsm_root', - return_value=ctsm_path): + with mock.patch("ctsm.path_utils.path_to_ctsm_root", return_value=ctsm_path): path_to_cime = path_utils.path_to_cime() self.assertEqual(path_to_cime, actual_path_to_cime) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_run_sys_tests.py b/python/ctsm/test/test_unit_run_sys_tests.py index 8a53081a5b..3ead0dbedd 100755 --- a/python/ctsm/test/test_unit_run_sys_tests.py +++ b/python/ctsm/test/test_unit_run_sys_tests.py @@ -14,7 +14,7 @@ import six from six_additions import mock, assertNotRegex -from ctsm import add_cime_to_path # pylint: disable=unused-import +from ctsm import add_cime_to_path # pylint: disable=unused-import from ctsm import unit_testing from ctsm.run_sys_tests import run_sys_tests from ctsm.machine_defaults import MACHINE_DEFAULTS @@ -26,17 +26,17 @@ # pylint: disable=invalid-name # Replace the slow _record_git_status with a fake that does nothing -@mock.patch('ctsm.run_sys_tests._record_git_status', mock.MagicMock(return_value=None)) +@mock.patch("ctsm.run_sys_tests._record_git_status", mock.MagicMock(return_value=None)) class TestRunSysTests(unittest.TestCase): """Tests of run_sys_tests""" - _MACHINE_NAME = 'fake_machine' + _MACHINE_NAME = "fake_machine" def setUp(self): self._original_wd = os.getcwd() self._curdir = tempfile.mkdtemp() os.chdir(self._curdir) - self._scratch = os.path.join(self._curdir, 'scratch') + self._scratch = os.path.join(self._curdir, "scratch") os.makedirs(self._scratch) def tearDown(self): @@ -44,51 +44,47 @@ def tearDown(self): shutil.rmtree(self._curdir, ignore_errors=True) def _make_machine(self, account=None): - machine = create_machine(machine_name=self._MACHINE_NAME, - defaults=MACHINE_DEFAULTS, - job_launcher_type=JOB_LAUNCHER_FAKE, - scratch_dir=self._scratch, - account=account) + machine = create_machine( + machine_name=self._MACHINE_NAME, + defaults=MACHINE_DEFAULTS, + job_launcher_type=JOB_LAUNCHER_FAKE, + scratch_dir=self._scratch, + account=account, + ) return machine @staticmethod def _fake_now(): - return datetime(year=2001, - month=2, - day=3, - hour=4, - minute=5, - second=6) + return datetime(year=2001, month=2, day=3, hour=4, minute=5, second=6) def _cime_path(self): # For the sake of paths to scripts used in run_sys_tests: Pretend that cime exists # under the current directory, even though it doesn't - return os.path.join(self._curdir, 'cime') + return os.path.join(self._curdir, "cime") @staticmethod def _expected_testid(): """Returns an expected testid based on values set in _fake_now and _MACHINE_NAME""" - return '0203-040506fa' + return "0203-040506fa" def _expected_testroot(self): """Returns an expected testroot based on values set in _fake_now and _MACHINE_NAME This just returns the name of the testroot directory, not the full path""" - return 'tests_{}'.format(self._expected_testid()) + return "tests_{}".format(self._expected_testid()) def test_testroot_setup(self): """Ensure that the appropriate test root directory is created and populated""" machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.datetime') as mock_date: + with mock.patch("ctsm.run_sys_tests.datetime") as mock_date: mock_date.now.side_effect = self._fake_now - run_sys_tests(machine=machine, cime_path=self._cime_path(), - testlist=['foo']) + run_sys_tests( + machine=machine, cime_path=self._cime_path(), testlist=["foo"] + ) - expected_dir = os.path.join(self._scratch, - self._expected_testroot()) + expected_dir = os.path.join(self._scratch, self._expected_testroot()) self.assertTrue(os.path.isdir(expected_dir)) - expected_link = os.path.join(self._curdir, - self._expected_testroot()) + expected_link = os.path.join(self._curdir, self._expected_testroot()) self.assertTrue(os.path.islink(expected_link)) self.assertEqual(os.readlink(expected_link), expected_dir) @@ -106,28 +102,37 @@ def test_createTestCommand_testnames(self): (3) That a cs.status.fails file was created """ machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.datetime') as mock_date: + with mock.patch("ctsm.run_sys_tests.datetime") as mock_date: mock_date.now.side_effect = self._fake_now - run_sys_tests(machine=machine, cime_path=self._cime_path(), - testlist=['test1', 'test2']) + run_sys_tests( + machine=machine, + cime_path=self._cime_path(), + testlist=["test1", "test2"], + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 1) command = all_commands[0].cmd - expected_create_test = os.path.join(self._cime_path(), 'scripts', 'create_test') - six.assertRegex(self, command, r'^ *{}\s'.format(re.escape(expected_create_test))) - six.assertRegex(self, command, r'--test-id +{}\s'.format(self._expected_testid())) + expected_create_test = os.path.join(self._cime_path(), "scripts", "create_test") + six.assertRegex( + self, command, r"^ *{}\s".format(re.escape(expected_create_test)) + ) + six.assertRegex( + self, command, r"--test-id +{}\s".format(self._expected_testid()) + ) expected_testroot_path = os.path.join(self._scratch, self._expected_testroot()) - six.assertRegex(self, command, r'--output-root +{}\s'.format(expected_testroot_path)) - six.assertRegex(self, command, r'--retry +0(\s|$)') - six.assertRegex(self, command, r'test1 +test2(\s|$)') - assertNotRegex(self, command, r'--compare\s') - assertNotRegex(self, command, r'--generate\s') - assertNotRegex(self, command, r'--baseline-root\s') - - expected_cs_status = os.path.join(self._scratch, - self._expected_testroot(), - 'cs.status.fails') + six.assertRegex( + self, command, r"--output-root +{}\s".format(expected_testroot_path) + ) + six.assertRegex(self, command, r"--retry +0(\s|$)") + six.assertRegex(self, command, r"test1 +test2(\s|$)") + assertNotRegex(self, command, r"--compare\s") + assertNotRegex(self, command, r"--generate\s") + assertNotRegex(self, command, r"--baseline-root\s") + + expected_cs_status = os.path.join( + self._scratch, self._expected_testroot(), "cs.status.fails" + ) self.assertTrue(os.path.isfile(expected_cs_status)) def test_createTestCommand_testfileAndExtraArgs(self): @@ -141,38 +146,42 @@ def test_createTestCommand_testfileAndExtraArgs(self): (3) That a cs.status.fails file was created """ - machine = self._make_machine(account='myaccount') - testroot_base = os.path.join(self._scratch, 'my', 'testroot') - run_sys_tests(machine=machine, cime_path=self._cime_path(), - testfile='/path/to/testfile', - testid_base='mytestid', - testroot_base=testroot_base, - compare_name='mycompare', - generate_name='mygenerate', - baseline_root='myblroot', - walltime='3:45:67', - queue='runqueue', - retry=5, - extra_create_test_args='--some extra --createtest args') + machine = self._make_machine(account="myaccount") + testroot_base = os.path.join(self._scratch, "my", "testroot") + run_sys_tests( + machine=machine, + cime_path=self._cime_path(), + testfile="/path/to/testfile", + testid_base="mytestid", + testroot_base=testroot_base, + compare_name="mycompare", + generate_name="mygenerate", + baseline_root="myblroot", + walltime="3:45:67", + queue="runqueue", + retry=5, + extra_create_test_args="--some extra --createtest args", + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 1) command = all_commands[0].cmd - six.assertRegex(self, command, r'--test-id +mytestid(\s|$)') - expected_testroot = os.path.join(testroot_base, 'tests_mytestid') - six.assertRegex(self, command, r'--output-root +{}(\s|$)'.format(expected_testroot)) - six.assertRegex(self, command, r'--testfile +/path/to/testfile(\s|$)') - six.assertRegex(self, command, r'--compare +mycompare(\s|$)') - six.assertRegex(self, command, r'--generate +mygenerate(\s|$)') - six.assertRegex(self, command, r'--baseline-root +myblroot(\s|$)') - six.assertRegex(self, command, r'--walltime +3:45:67(\s|$)') - six.assertRegex(self, command, r'--queue +runqueue(\s|$)') - six.assertRegex(self, command, r'--project +myaccount(\s|$)') - six.assertRegex(self, command, r'--retry +5(\s|$)') - six.assertRegex(self, command, r'--some +extra +--createtest +args(\s|$)') - - expected_cs_status = os.path.join(expected_testroot, - 'cs.status.fails') + six.assertRegex(self, command, r"--test-id +mytestid(\s|$)") + expected_testroot = os.path.join(testroot_base, "tests_mytestid") + six.assertRegex( + self, command, r"--output-root +{}(\s|$)".format(expected_testroot) + ) + six.assertRegex(self, command, r"--testfile +/path/to/testfile(\s|$)") + six.assertRegex(self, command, r"--compare +mycompare(\s|$)") + six.assertRegex(self, command, r"--generate +mygenerate(\s|$)") + six.assertRegex(self, command, r"--baseline-root +myblroot(\s|$)") + six.assertRegex(self, command, r"--walltime +3:45:67(\s|$)") + six.assertRegex(self, command, r"--queue +runqueue(\s|$)") + six.assertRegex(self, command, r"--project +myaccount(\s|$)") + six.assertRegex(self, command, r"--retry +5(\s|$)") + six.assertRegex(self, command, r"--some +extra +--createtest +args(\s|$)") + + expected_cs_status = os.path.join(expected_testroot, "cs.status.fails") self.assertTrue(os.path.isfile(expected_cs_status)) def test_createTestCommands_testsuite(self): @@ -186,77 +195,100 @@ def test_createTestCommands_testsuite(self): It also ensures that the cs.status.fails and cs.status files are created """ machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.datetime') as mock_date, \ - mock.patch('ctsm.run_sys_tests.get_tests_from_xml') as mock_get_tests: + with mock.patch("ctsm.run_sys_tests.datetime") as mock_date, mock.patch( + "ctsm.run_sys_tests.get_tests_from_xml" + ) as mock_get_tests: mock_date.now.side_effect = self._fake_now - mock_get_tests.return_value = [{'compiler': 'intel'}, - {'compiler': 'pgi'}, - {'compiler': 'intel'}] - run_sys_tests(machine=machine, cime_path=self._cime_path(), - suite_name='my_suite') + mock_get_tests.return_value = [ + {"compiler": "intel"}, + {"compiler": "pgi"}, + {"compiler": "intel"}, + ] + run_sys_tests( + machine=machine, cime_path=self._cime_path(), suite_name="my_suite" + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 2) for command in all_commands: - six.assertRegex(self, command.cmd, - r'--xml-category +{}(\s|$)'.format('my_suite')) - six.assertRegex(self, command.cmd, - r'--xml-machine +{}(\s|$)'.format(self._MACHINE_NAME)) - - six.assertRegex(self, all_commands[0].cmd, r'--xml-compiler +intel(\s|$)') - six.assertRegex(self, all_commands[1].cmd, r'--xml-compiler +pgi(\s|$)') - - expected_testid1 = '{}_int'.format(self._expected_testid()) - expected_testid2 = '{}_pgi'.format(self._expected_testid()) - six.assertRegex(self, all_commands[0].cmd, - r'--test-id +{}(\s|$)'.format(expected_testid1)) - six.assertRegex(self, all_commands[1].cmd, - r'--test-id +{}(\s|$)'.format(expected_testid2)) + six.assertRegex( + self, command.cmd, r"--xml-category +{}(\s|$)".format("my_suite") + ) + six.assertRegex( + self, command.cmd, r"--xml-machine +{}(\s|$)".format(self._MACHINE_NAME) + ) + + six.assertRegex(self, all_commands[0].cmd, r"--xml-compiler +intel(\s|$)") + six.assertRegex(self, all_commands[1].cmd, r"--xml-compiler +pgi(\s|$)") + + expected_testid1 = "{}_int".format(self._expected_testid()) + expected_testid2 = "{}_pgi".format(self._expected_testid()) + six.assertRegex( + self, all_commands[0].cmd, r"--test-id +{}(\s|$)".format(expected_testid1) + ) + six.assertRegex( + self, all_commands[1].cmd, r"--test-id +{}(\s|$)".format(expected_testid2) + ) expected_testroot_path = os.path.join(self._scratch, self._expected_testroot()) - self.assertEqual(all_commands[0].out, os.path.join(expected_testroot_path, - 'STDOUT.'+expected_testid1)) - self.assertEqual(all_commands[0].err, os.path.join(expected_testroot_path, - 'STDERR.'+expected_testid1)) - self.assertEqual(all_commands[1].out, os.path.join(expected_testroot_path, - 'STDOUT.'+expected_testid2)) - self.assertEqual(all_commands[1].err, os.path.join(expected_testroot_path, - 'STDERR.'+expected_testid2)) - - expected_cs_status = os.path.join(self._scratch, - self._expected_testroot(), - 'cs.status') - expected_cs_status = os.path.join(self._scratch, - self._expected_testroot(), - 'cs.status.fails') + self.assertEqual( + all_commands[0].out, + os.path.join(expected_testroot_path, "STDOUT." + expected_testid1), + ) + self.assertEqual( + all_commands[0].err, + os.path.join(expected_testroot_path, "STDERR." + expected_testid1), + ) + self.assertEqual( + all_commands[1].out, + os.path.join(expected_testroot_path, "STDOUT." + expected_testid2), + ) + self.assertEqual( + all_commands[1].err, + os.path.join(expected_testroot_path, "STDERR." + expected_testid2), + ) + + expected_cs_status = os.path.join( + self._scratch, self._expected_testroot(), "cs.status" + ) + expected_cs_status = os.path.join( + self._scratch, self._expected_testroot(), "cs.status.fails" + ) self.assertTrue(os.path.isfile(expected_cs_status)) def test_createTestCommands_testsuiteSpecifiedCompilers(self): """The correct commands should be run with a test suite where compilers are specified""" machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.get_tests_from_xml') as mock_get_tests: + with mock.patch("ctsm.run_sys_tests.get_tests_from_xml") as mock_get_tests: # This value should be ignored; we just set it to make sure it's different # from the passed-in compiler list - mock_get_tests.return_value = [{'compiler': 'intel'}, - {'compiler': 'pgi'}, - {'compiler': 'gnu'}] - run_sys_tests(machine=machine, cime_path=self._cime_path(), - suite_name='my_suite', - suite_compilers=['comp1a', 'comp2b']) + mock_get_tests.return_value = [ + {"compiler": "intel"}, + {"compiler": "pgi"}, + {"compiler": "gnu"}, + ] + run_sys_tests( + machine=machine, + cime_path=self._cime_path(), + suite_name="my_suite", + suite_compilers=["comp1a", "comp2b"], + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 2) - six.assertRegex(self, all_commands[0].cmd, r'--xml-compiler +comp1a(\s|$)') - six.assertRegex(self, all_commands[1].cmd, r'--xml-compiler +comp2b(\s|$)') + six.assertRegex(self, all_commands[0].cmd, r"--xml-compiler +comp1a(\s|$)") + six.assertRegex(self, all_commands[1].cmd, r"--xml-compiler +comp2b(\s|$)") def test_withDryRun_nothingDone(self): """With dry_run=True, no directories should be created, and no commands should be run""" machine = self._make_machine() - run_sys_tests(machine=machine, cime_path=self._cime_path(), testlist=['foo'], - dry_run=True) + run_sys_tests( + machine=machine, cime_path=self._cime_path(), testlist=["foo"], dry_run=True + ) self.assertEqual(os.listdir(self._scratch), []) self.assertEqual(machine.job_launcher.get_commands(), []) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index 4a3fbbbb15..8ba8bf54c1 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -15,6 +15,7 @@ # to make readable unit test names # pylint: disable=invalid-name + class TestUtilsFillTemplateFile(unittest.TestCase): """Tests of utils: fill_template_file""" @@ -26,19 +27,18 @@ def tearDown(self): def test_fillTemplateFile_basic(self): """Basic test of fill_template_file""" - template_path = os.path.join(self._testdir, 'template.txt') - final_path = os.path.join(self._testdir, 'final.txt') + template_path = os.path.join(self._testdir, "template.txt") + final_path = os.path.join(self._testdir, "final.txt") template_contents = """\ Hello $foo Goodbye $bar """ - with open(template_path, 'w') as f: + with open(template_path, "w") as f: f.write(template_contents) - fillins = {'foo':'aardvark', - 'bar':'zyzzyva'} + fillins = {"foo": "aardvark", "bar": "zyzzyva"} fill_template_file(template_path, final_path, fillins) expected_final_text = """\ @@ -52,6 +52,7 @@ def test_fillTemplateFile_basic(self): self.assertEqual(final_contents, expected_final_text) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/unit_testing.py b/python/ctsm/unit_testing.py index 7d8f40173a..d3a308c796 100644 --- a/python/ctsm/unit_testing.py +++ b/python/ctsm/unit_testing.py @@ -2,6 +2,7 @@ from ctsm.ctsm_logging import setup_logging_for_tests + def setup_for_tests(enable_critical_logs=False): """Call this at the beginning of unit testing diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 44cce0cccf..3320b1ebc8 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + def abort(errmsg): """Abort the program with the given error message @@ -15,7 +16,8 @@ def abort(errmsg): if logger.isEnabledFor(logging.DEBUG): pdb.set_trace() - sys.exit('ERROR: {}'.format(errmsg)) + sys.exit("ERROR: {}".format(errmsg)) + def fill_template_file(path_to_template, path_to_final, substitutions): """Given a template file (based on python's template strings), write a copy of the @@ -31,5 +33,5 @@ def fill_template_file(path_to_template, path_to_final, substitutions): template_file_contents = template_file.read() template = string.Template(template_file_contents) final_file_contents = template.substitute(substitutions) - with open(path_to_final, 'w') as final_file: + with open(path_to_final, "w") as final_file: final_file.write(final_file_contents) diff --git a/python/six.py b/python/six.py index 6bf4fd3810..df4958f3b8 100644 --- a/python/six.py +++ b/python/six.py @@ -38,15 +38,15 @@ PY34 = sys.version_info[0:2] >= (3, 4) if PY3: - string_types = str, - integer_types = int, - class_types = type, + string_types = (str,) + integer_types = (int,) + class_types = (type,) text_type = str binary_type = bytes MAXSIZE = sys.maxsize else: - string_types = basestring, + string_types = (basestring,) integer_types = (int, long) class_types = (type, types.ClassType) text_type = unicode @@ -58,9 +58,9 @@ else: # It's possible to have sizeof(long) != sizeof(Py_ssize_t). class X(object): - def __len__(self): return 1 << 31 + try: len(X()) except OverflowError: @@ -84,7 +84,6 @@ def _import_module(name): class _LazyDescr(object): - def __init__(self, name): self.name = name @@ -101,7 +100,6 @@ def __get__(self, obj, tp): class MovedModule(_LazyDescr): - def __init__(self, name, old, new=None): super(MovedModule, self).__init__(name) if PY3: @@ -122,7 +120,6 @@ def __getattr__(self, attr): class _LazyModule(types.ModuleType): - def __init__(self, name): super(_LazyModule, self).__init__(name) self.__doc__ = self.__class__.__doc__ @@ -137,7 +134,6 @@ def __dir__(self): class MovedAttribute(_LazyDescr): - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): super(MovedAttribute, self).__init__(name) if PY3: @@ -221,21 +217,26 @@ def get_code(self, fullname): Required, if is_package is implemented""" self.__get_module(fullname) # eventually raises ImportError return None + get_source = get_code # same as get_code + _importer = _SixMetaPathImporter(__name__) class _MovedItems(_LazyModule): """Lazy loading of moved objects""" + __path__ = [] # mark as package _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute( + "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" + ), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), @@ -243,7 +244,9 @@ class _MovedItems(_LazyModule): MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute( + "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" + ), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), @@ -252,7 +255,9 @@ class _MovedItems(_LazyModule): MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedAttribute( + "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" + ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), @@ -266,7 +271,9 @@ class _MovedItems(_LazyModule): MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule( + "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" + ), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), @@ -285,15 +292,12 @@ class _MovedItems(_LazyModule): MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), + MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"), MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), MovedModule("tkinter_font", "tkFont", "tkinter.font"), MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), @@ -339,7 +343,9 @@ class Module_six_moves_urllib_parse(_LazyModule): MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute( + "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes" + ), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), @@ -357,8 +363,11 @@ class Module_six_moves_urllib_parse(_LazyModule): Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") +_importer._add_module( + Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", + "moves.urllib.parse", +) class Module_six_moves_urllib_error(_LazyModule): @@ -377,8 +386,11 @@ class Module_six_moves_urllib_error(_LazyModule): Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") +_importer._add_module( + Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", + "moves.urllib.error", +) class Module_six_moves_urllib_request(_LazyModule): @@ -429,8 +441,11 @@ class Module_six_moves_urllib_request(_LazyModule): Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") +_importer._add_module( + Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", + "moves.urllib.request", +) class Module_six_moves_urllib_response(_LazyModule): @@ -450,8 +465,11 @@ class Module_six_moves_urllib_response(_LazyModule): Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") +_importer._add_module( + Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", + "moves.urllib.response", +) class Module_six_moves_urllib_robotparser(_LazyModule): @@ -466,15 +484,21 @@ class Module_six_moves_urllib_robotparser(_LazyModule): setattr(Module_six_moves_urllib_robotparser, attr.name, attr) del attr -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes +Module_six_moves_urllib_robotparser._moved_attributes = ( + _urllib_robotparser_moved_attributes +) -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") +_importer._add_module( + Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", + "moves.urllib.robotparser", +) class Module_six_moves_urllib(types.ModuleType): """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package parse = _importer._get_module("moves.urllib_parse") error = _importer._get_module("moves.urllib_error") @@ -483,10 +507,12 @@ class Module_six_moves_urllib(types.ModuleType): robotparser = _importer._get_module("moves.urllib_robotparser") def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] + return ["parse", "error", "request", "response", "robotparser"] + -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") +_importer._add_module( + Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" +) def add_move(move): @@ -526,19 +552,24 @@ def remove_move(name): try: advance_iterator = next except NameError: + def advance_iterator(it): return it.next() + + next = advance_iterator try: callable = callable except NameError: + def callable(obj): return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) if PY3: + def get_unbound_function(unbound): return unbound @@ -549,6 +580,7 @@ def create_unbound_method(func, cls): Iterator = object else: + def get_unbound_function(unbound): return unbound.im_func @@ -559,13 +591,13 @@ def create_unbound_method(func, cls): return types.MethodType(func, None, cls) class Iterator(object): - def next(self): return type(self).__next__(self) callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") +_add_doc( + get_unbound_function, """Get the function out of a possibly unbound function""" +) get_method_function = operator.attrgetter(_meth_func) @@ -577,6 +609,7 @@ def next(self): if PY3: + def iterkeys(d, **kw): return iter(d.keys(**kw)) @@ -595,6 +628,7 @@ def iterlists(d, **kw): viewitems = operator.methodcaller("items") else: + def iterkeys(d, **kw): return d.iterkeys(**kw) @@ -615,26 +649,30 @@ def iterlists(d, **kw): _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") _add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") +_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc( + iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." +) if PY3: + def b(s): return s.encode("latin-1") def u(s): return s + unichr = chr import struct + int2byte = struct.Struct(">B").pack del struct byte2int = operator.itemgetter(0) indexbytes = operator.getitem iterbytes = iter import io + StringIO = io.StringIO BytesIO = io.BytesIO _assertCountEqual = "assertCountEqual" @@ -645,12 +683,15 @@ def u(s): _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" else: + def b(s): return s + # Workaround for standalone backslash def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") + unichr = unichr int2byte = chr @@ -659,8 +700,10 @@ def byte2int(bs): def indexbytes(buf, i): return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) import StringIO + StringIO = BytesIO = StringIO.StringIO _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" @@ -695,7 +738,9 @@ def reraise(tp, value, tb=None): value = None tb = None + else: + def exec_(_code_, _globs_=None, _locs_=None): """Execute code in a namespace.""" if _globs_ is None: @@ -708,37 +753,45 @@ def exec_(_code_, _globs_=None, _locs_=None): _locs_ = _globs_ exec("""exec _code_ in _globs_, _locs_""") - exec_("""def reraise(tp, value, tb=None): + exec_( + """def reraise(tp, value, tb=None): try: raise tp, value, tb finally: tb = None -""") +""" + ) if sys.version_info[:2] == (3, 2): - exec_("""def raise_from(value, from_value): + exec_( + """def raise_from(value, from_value): try: if from_value is None: raise value raise value from from_value finally: value = None -""") +""" + ) elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): + exec_( + """def raise_from(value, from_value): try: raise value from from_value finally: value = None -""") +""" + ) else: + def raise_from(value, from_value): raise value print_ = getattr(moves.builtins, "print", None) if print_ is None: + def print_(*args, **kwargs): """The new-style print function for Python 2.4 and 2.5.""" fp = kwargs.pop("file", sys.stdout) @@ -749,14 +802,17 @@ def write(data): if not isinstance(data, basestring): data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): + if ( + isinstance(fp, file) + and isinstance(data, unicode) + and fp.encoding is not None + ): errors = getattr(fp, "errors", None) if errors is None: errors = "strict" data = data.encode(fp.encoding, errors) fp.write(data) + want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: @@ -792,6 +848,8 @@ def write(data): write(sep) write(arg) write(end) + + if sys.version_info[:2] < (3, 3): _print = print_ @@ -802,16 +860,24 @@ def print_(*args, **kwargs): if flush and fp is not None: fp.flush() + _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): + + def wraps( + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): def wrapper(f): f = functools.wraps(wrapped, assigned, updated)(f) f.__wrapped__ = wrapped return f + return wrapper + + else: wraps = functools.wraps @@ -822,29 +888,31 @@ def with_metaclass(meta, *bases): # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. class metaclass(type): - def __new__(cls, name, this_bases, d): return meta(name, bases, d) @classmethod def __prepare__(cls, name, this_bases): return meta.__prepare__(name, bases) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) def add_metaclass(metaclass): """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') + slots = orig_vars.get("__slots__") if slots is not None: if isinstance(slots, str): slots = [slots] for slots_var in slots: orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) + orig_vars.pop("__dict__", None) + orig_vars.pop("__weakref__", None) return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper @@ -857,12 +925,13 @@ def python_2_unicode_compatible(klass): returning text and apply this decorator to the class. """ if PY2: - if '__str__' not in klass.__dict__: - raise ValueError("@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % - klass.__name__) + if "__str__" not in klass.__dict__: + raise ValueError( + "@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % klass.__name__ + ) klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + klass.__str__ = lambda self: self.__unicode__().encode("utf-8") return klass @@ -882,8 +951,10 @@ def python_2_unicode_compatible(klass): # be floating around. Therefore, we can't use isinstance() to check for # the six meta path importer, since the other six instance will have # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): + if ( + type(importer).__name__ == "_SixMetaPathImporter" + and importer.name == __name__ + ): del sys.meta_path[i] break del i, importer diff --git a/python/six_additions.py b/python/six_additions.py index db1003f36c..0934530d54 100644 --- a/python/six_additions.py +++ b/python/six_additions.py @@ -16,5 +16,6 @@ else: _assertNotRegex = "assertNotRegexpMatches" + def assertNotRegex(self, *args, **kwargs): return getattr(self, _assertNotRegex)(*args, **kwargs) From 5b1d9b935e4b48c02b865b2c3c7f46c437eab45a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 13 Dec 2021 01:37:27 -0700 Subject: [PATCH 02/16] Add auto black check of python code under python directory on push --- .github/workflows/black.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/black.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 0000000000..5c8eef7254 --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,19 @@ +name: push black workflow +# +# Run the python formatting in check mode +# +on: push + +jobs: + black-check: + runs-on: ubuntu-latest + steps: + # Checkout the code + - uses: actions/checkout@v2 + # Use a specific version of the black-checker + # Use version that's the same as with conda on cheyenne + # Tag for black-check corresponds to the black version tag + - uses: jpetrucciani/black-check@21.11b1 + with: + path: python + black_flags: --config python/pyproject.toml From 21e2be3275b03bd823f87c77dd54e608565e5fe3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 13 Dec 2021 01:49:53 -0700 Subject: [PATCH 03/16] Get pylint to work with newer black --- python/ctsm/.pylintrc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/ctsm/.pylintrc b/python/ctsm/.pylintrc index bc7ae54dd2..f702d57814 100644 --- a/python/ctsm/.pylintrc +++ b/python/ctsm/.pylintrc @@ -140,6 +140,11 @@ disable=print-statement, deprecated-sys-function, exception-escape, comprehension-escape, + R0801, # This option to the end are options required to use black formatter + W1515, + W1514, + R1732, + C0209, C0330, # This is an option that the formatter "black" requires us to disable # --- default list is above here, our own list is below here --- # While pylint's recommendations to keep the number of arguments, local From b10b3c44a61da4ff2a4f276ffdd914a0dcbb92c8 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 14 Dec 2021 00:00:28 -0700 Subject: [PATCH 04/16] Add git blame ignore revision file --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..517bd4cc8c --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Ran python directory through black python formatter +2cdb380febb274478e84cd90945aee93f29fa2e6 From 75f290c6c07d45f742d9cea57136baf9ddfa498a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 14 Dec 2021 00:08:45 -0700 Subject: [PATCH 05/16] Make changes suggested by black in regard to pylint --- python/ctsm/.pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ctsm/.pylintrc b/python/ctsm/.pylintrc index f702d57814..a6be427a19 100644 --- a/python/ctsm/.pylintrc +++ b/python/ctsm/.pylintrc @@ -145,6 +145,7 @@ disable=print-statement, W1514, R1732, C0209, + C0326, # This and the next are the two options that black documentation tells us to ignore C0330, # This is an option that the formatter "black" requires us to disable # --- default list is above here, our own list is below here --- # While pylint's recommendations to keep the number of arguments, local @@ -334,7 +335,7 @@ indent-after-paren=4 indent-string=' ' # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=88 # Make sure this agrees with black # Maximum number of lines in a module. max-module-lines=1000 From 0aa2957c1f8603c63fa30b11295c06cfddff44a5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Mar 2022 17:32:19 -0700 Subject: [PATCH 06/16] Run through black 22.1.0 --- python/ctsm/args_utils.py | 2 + python/ctsm/gen_mksurf_namelist.py | 2 +- python/ctsm/lilac_make_runtime_inputs.py | 274 ++++--- python/ctsm/machine.py | 98 ++- python/ctsm/machine_defaults.py | 88 +- python/ctsm/machine_utils.py | 12 +- .../ctsm/modify_fsurdat/fsurdat_modifier.py | 222 +++-- python/ctsm/modify_fsurdat/modify_fsurdat.py | 158 ++-- python/ctsm/run_sys_tests.py | 764 +++++++++++------- python/ctsm/subset_data.py | 196 +++-- python/ctsm/test/test_sys_fsurdat_modifier.py | 162 ++-- python/ctsm/test/test_unit_args_utils.py | 9 +- python/ctsm/test/test_unit_machine.py | 254 +++--- python/ctsm/test/test_unit_modify_fsurdat.py | 71 +- python/ctsm/test/test_unit_run_sys_tests.py | 266 +++--- python/ctsm/test/test_unit_utils.py | 175 ++-- python/ctsm/toolchain/ctsm_case.py | 32 +- python/ctsm/utils.py | 1 + python/six.py | 2 - 19 files changed, 1685 insertions(+), 1103 deletions(-) diff --git a/python/ctsm/args_utils.py b/python/ctsm/args_utils.py index d944227157..357d6a41da 100644 --- a/python/ctsm/args_utils.py +++ b/python/ctsm/args_utils.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) + def plat_type(plat): """ Function to define lat type for the parser @@ -31,6 +32,7 @@ def plat_type(plat): ) return plat_out + def plon_type(plon): """ Function to define lon type for the parser and diff --git a/python/ctsm/gen_mksurf_namelist.py b/python/ctsm/gen_mksurf_namelist.py index 34acd110e2..3b846c2c39 100644 --- a/python/ctsm/gen_mksurf_namelist.py +++ b/python/ctsm/gen_mksurf_namelist.py @@ -363,7 +363,7 @@ def main(): glc_flag, start_year, end_year, - hres_flag + hres_flag, ) logger.info("--------------------------") diff --git a/python/ctsm/lilac_make_runtime_inputs.py b/python/ctsm/lilac_make_runtime_inputs.py index 1d156077d9..f9f8dcd9b4 100644 --- a/python/ctsm/lilac_make_runtime_inputs.py +++ b/python/ctsm/lilac_make_runtime_inputs.py @@ -7,9 +7,13 @@ from configparser import ConfigParser -from CIME.buildnml import create_namelist_infile # pylint: disable=import-error +from CIME.buildnml import create_namelist_infile # pylint: disable=import-error -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) from ctsm.path_utils import path_to_ctsm_root from ctsm.utils import abort from ctsm.config_utils import get_config_value @@ -31,7 +35,7 @@ # Note the following is needed in env_lilac.xml otherwise the following error appears in # the call to build_namelist -#err=ERROR : CLM build-namelist::CLMBuildNamelist::logical_to_fortran() : +# err=ERROR : CLM build-namelist::CLMBuildNamelist::logical_to_fortran() : # Unexpected value in logical_to_fortran: _ENV_LILAC_TEMPLATE = """ @@ -51,8 +55,10 @@ # case object # ======================================================================== + class CaseFake: """Fake case class to satisfy interface of CIME functions that need a case object""" + # pylint: disable=too-few-public-methods def __init__(self): @@ -66,9 +72,10 @@ def get_resolved_value(value): """ abort("Cannot resolve value with a '$' variable: {}".format(value)) + ############################################################################### def parse_command_line(): -############################################################################### + ############################################################################### """Parse the command line, return object holding arguments""" @@ -76,11 +83,16 @@ def parse_command_line(): Script to create runtime inputs when running CTSM via LILAC """ - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, - description=description) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, description=description + ) - parser.add_argument("--rundir", type=str, default=os.getcwd(), - help="Full path of the run directory (containing ctsm.cfg & user_nl_ctsm)") + parser.add_argument( + "--rundir", + type=str, + default=os.getcwd(), + help="Full path of the run directory (containing ctsm.cfg & user_nl_ctsm)", + ) add_logging_args(parser) @@ -93,93 +105,133 @@ def parse_command_line(): return arguments + ############################################################################### def determine_bldnml_opts(bgc_mode, crop, vichydro): -############################################################################### + ############################################################################### """Return a string giving bldnml options, given some other inputs""" - bldnml_opts = '' - bldnml_opts += ' -bgc {}'.format(bgc_mode) - if bgc_mode == 'fates': + bldnml_opts = "" + bldnml_opts += " -bgc {}".format(bgc_mode) + if bgc_mode == "fates": # BUG(wjs, 2020-06-12, ESCOMP/CTSM#115) For now, FATES is incompatible with MEGAN - bldnml_opts += ' -no-megan' - - if crop == 'on': - if bgc_mode not in ['bgc', 'cn']: - abort("Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'") - bldnml_opts += ' -crop' - - if vichydro == 'on': - if bgc_mode != 'sp': - abort("Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'") - bldnml_opts += ' -vichydro' + bldnml_opts += " -no-megan" + + if crop == "on": + if bgc_mode not in ["bgc", "cn"]: + abort( + "Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'" + ) + bldnml_opts += " -crop" + + if vichydro == "on": + if bgc_mode != "sp": + abort( + "Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'" + ) + bldnml_opts += " -vichydro" return bldnml_opts + ############################################################################### def buildnml(cime_path, rundir): -############################################################################### + ############################################################################### - """Build the ctsm namelist - """ + """Build the ctsm namelist""" # pylint: disable=too-many-locals # pylint: disable=too-many-statements - ctsm_cfg_path = os.path.join(rundir, 'ctsm.cfg') + ctsm_cfg_path = os.path.join(rundir, "ctsm.cfg") # read the config file config = ConfigParser() config.read(ctsm_cfg_path) - lnd_domain_file = get_config_value(config, 'buildnml_input', 'lnd_domain_file', ctsm_cfg_path) - fsurdat = get_config_value(config, 'buildnml_input', 'fsurdat', - ctsm_cfg_path, can_be_unset=True) - finidat = get_config_value(config, 'buildnml_input', 'finidat', - ctsm_cfg_path, can_be_unset=True) - - ctsm_phys = get_config_value(config, 'buildnml_input', 'ctsm_phys', ctsm_cfg_path, - allowed_values=['clm4_5', 'clm5_0', 'clm5_1']) - configuration = get_config_value(config, 'buildnml_input', 'configuration', ctsm_cfg_path, - allowed_values=['nwp', 'clm']) - structure = get_config_value(config, 'buildnml_input', 'structure', ctsm_cfg_path, - allowed_values=['fast', 'standard']) - bgc_mode = get_config_value(config, 'buildnml_input', 'bgc_mode', ctsm_cfg_path, - allowed_values=['sp', 'bgc', 'cn', 'fates']) - crop = get_config_value(config, 'buildnml_input', 'crop', ctsm_cfg_path, - allowed_values=['off', 'on']) - vichydro = get_config_value(config, 'buildnml_input', 'vichydro', ctsm_cfg_path, - allowed_values=['off', 'on']) - - bldnml_opts = determine_bldnml_opts(bgc_mode=bgc_mode, - crop=crop, - vichydro=vichydro) - - co2_ppmv = get_config_value(config, 'buildnml_input', 'co2_ppmv', ctsm_cfg_path) - use_case = get_config_value(config, 'buildnml_input', 'use_case', ctsm_cfg_path) - lnd_tuning_mode = get_config_value(config, 'buildnml_input', 'lnd_tuning_mode', ctsm_cfg_path) - spinup = get_config_value(config, 'buildnml_input', 'spinup', ctsm_cfg_path, - allowed_values=['off', 'on']) - - inputdata_path = get_config_value(config, 'buildnml_input', 'inputdata_path', ctsm_cfg_path) + lnd_domain_file = get_config_value( + config, "buildnml_input", "lnd_domain_file", ctsm_cfg_path + ) + fsurdat = get_config_value( + config, "buildnml_input", "fsurdat", ctsm_cfg_path, can_be_unset=True + ) + finidat = get_config_value( + config, "buildnml_input", "finidat", ctsm_cfg_path, can_be_unset=True + ) + + ctsm_phys = get_config_value( + config, + "buildnml_input", + "ctsm_phys", + ctsm_cfg_path, + allowed_values=["clm4_5", "clm5_0", "clm5_1"], + ) + configuration = get_config_value( + config, + "buildnml_input", + "configuration", + ctsm_cfg_path, + allowed_values=["nwp", "clm"], + ) + structure = get_config_value( + config, + "buildnml_input", + "structure", + ctsm_cfg_path, + allowed_values=["fast", "standard"], + ) + bgc_mode = get_config_value( + config, + "buildnml_input", + "bgc_mode", + ctsm_cfg_path, + allowed_values=["sp", "bgc", "cn", "fates"], + ) + crop = get_config_value( + config, "buildnml_input", "crop", ctsm_cfg_path, allowed_values=["off", "on"] + ) + vichydro = get_config_value( + config, + "buildnml_input", + "vichydro", + ctsm_cfg_path, + allowed_values=["off", "on"], + ) + + bldnml_opts = determine_bldnml_opts(bgc_mode=bgc_mode, crop=crop, vichydro=vichydro) + + co2_ppmv = get_config_value(config, "buildnml_input", "co2_ppmv", ctsm_cfg_path) + use_case = get_config_value(config, "buildnml_input", "use_case", ctsm_cfg_path) + lnd_tuning_mode = get_config_value( + config, "buildnml_input", "lnd_tuning_mode", ctsm_cfg_path + ) + spinup = get_config_value( + config, "buildnml_input", "spinup", ctsm_cfg_path, allowed_values=["off", "on"] + ) + + inputdata_path = get_config_value( + config, "buildnml_input", "inputdata_path", ctsm_cfg_path + ) # Parse the user_nl_ctsm file - infile = os.path.join(rundir, '.namelist') - create_namelist_infile(case=CaseFake(), - user_nl_file=os.path.join(rundir, 'user_nl_ctsm'), - namelist_infile=infile) + infile = os.path.join(rundir, ".namelist") + create_namelist_infile( + case=CaseFake(), + user_nl_file=os.path.join(rundir, "user_nl_ctsm"), + namelist_infile=infile, + ) # create config_cache.xml file # Note that build-namelist utilizes the contents of the config_cache.xml file in # the namelist_defaults.xml file to obtain namelist variables config_cache = os.path.join(rundir, "config_cache.xml") config_cache_text = _CONFIG_CACHE_TEMPLATE.format(clm_phys=ctsm_phys) - with open(config_cache, 'w') as tempfile: + with open(config_cache, "w") as tempfile: tempfile.write(config_cache_text) # create temporary env_lilac.xml env_lilac = os.path.join(rundir, "env_lilac.xml") env_lilac_text = _ENV_LILAC_TEMPLATE.format() - with open(env_lilac, 'w') as tempfile: + with open(env_lilac, "w") as tempfile: tempfile.write(env_lilac_text) # remove any existing clm.input_data_list file @@ -188,7 +240,7 @@ def buildnml(cime_path, rundir): os.remove(inputdatalist_path) # determine if fsurdat and/or finidat should appear in the -namelist option - extra_namelist_opts = '' + extra_namelist_opts = "" if fsurdat is not None: # NOTE(wjs, 2020-06-30) With the current logic, fsurdat should never be UNSET # (ie None here) but it's possible that this will change in the future. @@ -198,45 +250,63 @@ def buildnml(cime_path, rundir): # call build-namelist cmd = os.path.abspath(os.path.join(path_to_ctsm_root(), "bld", "build-namelist")) - command = [cmd, - '-driver', 'nuopc', - '-cimeroot', cime_path, - '-infile', infile, - '-csmdata', inputdata_path, - '-inputdata', inputdatalist_path, - # Hard-code start_ymd of year-2000. This is used to set the run type (for - # which a setting of 2000 gives 'startup', which is what we want) and pick - # the initial conditions file (which is pretty much irrelevant when running - # with lilac). - '-namelist', '&clm_inparm start_ymd=20000101 {} /'.format(extra_namelist_opts), - '-use_case', use_case, - # For now, we assume ignore_ic_year, not ignore_ic_date - '-ignore_ic_year', - # -clm_start_type seems unimportant (see discussion in - # https://github.com/ESCOMP/CTSM/issues/876) - '-clm_start_type', 'default', - '-configuration', configuration, - '-structure', structure, - '-lilac', - '-lnd_frac', lnd_domain_file, - '-glc_nec', str(10), - '-co2_ppmv', co2_ppmv, - '-co2_type', 'constant', - '-clm_accelerated_spinup', spinup, - '-lnd_tuning_mode', lnd_tuning_mode, - # Eventually make -no-megan dynamic (see - # https://github.com/ESCOMP/CTSM/issues/926) - '-no-megan', - '-config', os.path.join(rundir, "config_cache.xml"), - '-envxml_dir', rundir] + command = [ + cmd, + "-driver", + "nuopc", + "-cimeroot", + cime_path, + "-infile", + infile, + "-csmdata", + inputdata_path, + "-inputdata", + inputdatalist_path, + # Hard-code start_ymd of year-2000. This is used to set the run type (for + # which a setting of 2000 gives 'startup', which is what we want) and pick + # the initial conditions file (which is pretty much irrelevant when running + # with lilac). + "-namelist", + "&clm_inparm start_ymd=20000101 {} /".format(extra_namelist_opts), + "-use_case", + use_case, + # For now, we assume ignore_ic_year, not ignore_ic_date + "-ignore_ic_year", + # -clm_start_type seems unimportant (see discussion in + # https://github.com/ESCOMP/CTSM/issues/876) + "-clm_start_type", + "default", + "-configuration", + configuration, + "-structure", + structure, + "-lilac", + "-lnd_frac", + lnd_domain_file, + "-glc_nec", + str(10), + "-co2_ppmv", + co2_ppmv, + "-co2_type", + "constant", + "-clm_accelerated_spinup", + spinup, + "-lnd_tuning_mode", + lnd_tuning_mode, + # Eventually make -no-megan dynamic (see + # https://github.com/ESCOMP/CTSM/issues/926) + "-no-megan", + "-config", + os.path.join(rundir, "config_cache.xml"), + "-envxml_dir", + rundir, + ] # NOTE(wjs, 2020-06-16) Note that we do NOT use the -mask argument; it's possible that # we should be using it in some circumstances (I haven't looked into how it's used). - command.extend(['-res', 'lilac', - '-clm_usr_name', 'lilac']) + command.extend(["-res", "lilac", "-clm_usr_name", "lilac"]) command.extend(bldnml_opts.split()) - subprocess.check_call(command, - universal_newlines=True) + subprocess.check_call(command, universal_newlines=True) # remove temporary files in rundir os.remove(os.path.join(rundir, "config_cache.xml")) @@ -244,6 +314,7 @@ def buildnml(cime_path, rundir): os.remove(os.path.join(rundir, "drv_flds_in")) os.remove(infile) + ############################################################################### def main(cime_path): """Main function @@ -258,8 +329,7 @@ def main(cime_path): args = parse_command_line() process_logging_args(args) - buildnml( - cime_path=cime_path, - rundir=args.rundir) + buildnml(cime_path=cime_path, rundir=args.rundir) + ############################################################################### diff --git a/python/ctsm/machine.py b/python/ctsm/machine.py index 607e0b43af..fec7908cb0 100644 --- a/python/ctsm/machine.py +++ b/python/ctsm/machine.py @@ -4,8 +4,10 @@ import logging from collections import namedtuple from CIME.utils import get_project # pylint: disable=import-error -from ctsm.joblauncher.job_launcher_factory import \ - create_job_launcher, JOB_LAUNCHER_NOBATCH +from ctsm.joblauncher.job_launcher_factory import ( + create_job_launcher, + JOB_LAUNCHER_NOBATCH, +) # Value of create_test_queue for which we don't actually add a '--queue' option to # create_test, but instead leave that value unspecified, allowing CIME to pick an @@ -28,20 +30,32 @@ # user of the machine object to check for that possibility if need be. # # Similar notes apply to baseline_dir. -Machine = namedtuple('Machine', ['name', # str - 'scratch_dir', # str - 'baseline_dir', # str - 'account', # str or None - 'create_test_retry', # int - 'create_test_queue', # str - 'job_launcher']) # subclass of JobLauncherBase - -def create_machine(machine_name, defaults, job_launcher_type=None, - scratch_dir=None, account=None, - job_launcher_queue=None, job_launcher_walltime=None, - job_launcher_nice_level=None, - job_launcher_extra_args=None, - allow_missing_entries=False): +Machine = namedtuple( + "Machine", + [ + "name", # str + "scratch_dir", # str + "baseline_dir", # str + "account", # str or None + "create_test_retry", # int + "create_test_queue", # str + "job_launcher", + ], +) # subclass of JobLauncherBase + + +def create_machine( + machine_name, + defaults, + job_launcher_type=None, + scratch_dir=None, + account=None, + job_launcher_queue=None, + job_launcher_walltime=None, + job_launcher_nice_level=None, + job_launcher_extra_args=None, + allow_missing_entries=False, +): """Create a machine object (of type Machine, as given above) This uses the provided (non-None) arguments to override any defaults provided via the @@ -108,14 +122,20 @@ def create_machine(machine_name, defaults, job_launcher_type=None, # in a particular case. create_test_retry = mach_defaults.create_test_retry create_test_queue = mach_defaults.create_test_queue - if account is None and mach_defaults.account_required and not allow_missing_entries: + if ( + account is None + and mach_defaults.account_required + and not allow_missing_entries + ): raise RuntimeError("Could not find an account code") else: if not allow_missing_entries: # This isn't exactly a missing entry, but the times we don't care about this # warning tend to be the same as the times when allow_missing_entries is true - logger.warning("machine %s not recognized; using generic no-batch settings", - machine_name) + logger.warning( + "machine %s not recognized; using generic no-batch settings", + machine_name, + ) if job_launcher_type is None: job_launcher_type = JOB_LAUNCHER_NOBATCH @@ -127,7 +147,7 @@ def create_machine(machine_name, defaults, job_launcher_type=None, # sense to have these in the argument list for this function: they should only come # from the defaults structure. If the user wants to provide their own arguments, those # should be provided via extra_args. - job_launcher_required_args = '' + job_launcher_required_args = "" if mach_defaults is not None: these_defaults = mach_defaults.job_launcher_defaults.get(job_launcher_type) @@ -144,22 +164,27 @@ def create_machine(machine_name, defaults, job_launcher_type=None, # Create the job launcher and the full machine object # ------------------------------------------------------------------------ - job_launcher = create_job_launcher(job_launcher_type=job_launcher_type, - account=account, - queue=job_launcher_queue, - walltime=job_launcher_walltime, - nice_level=job_launcher_nice_level, - required_args=job_launcher_required_args, - extra_args=job_launcher_extra_args, - allow_missing_entries=allow_missing_entries) - - return Machine(name=machine_name, - scratch_dir=scratch_dir, - baseline_dir=baseline_dir, - account=account, - create_test_retry=create_test_retry, - create_test_queue=create_test_queue, - job_launcher=job_launcher) + job_launcher = create_job_launcher( + job_launcher_type=job_launcher_type, + account=account, + queue=job_launcher_queue, + walltime=job_launcher_walltime, + nice_level=job_launcher_nice_level, + required_args=job_launcher_required_args, + extra_args=job_launcher_extra_args, + allow_missing_entries=allow_missing_entries, + ) + + return Machine( + name=machine_name, + scratch_dir=scratch_dir, + baseline_dir=baseline_dir, + account=account, + create_test_retry=create_test_retry, + create_test_queue=create_test_queue, + job_launcher=job_launcher, + ) + def get_possibly_overridden_mach_value(machine, varname, value=None): """Get the value to use for the given machine variable @@ -176,6 +201,7 @@ def get_possibly_overridden_mach_value(machine, varname, value=None): return value return getattr(machine, varname) + def _get_account(): account = get_project() return account diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index 6b387741d5..e31ea9584d 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -4,18 +4,22 @@ from collections import namedtuple import os -from ctsm.joblauncher.job_launcher_factory import \ - JOB_LAUNCHER_QSUB +from ctsm.joblauncher.job_launcher_factory import JOB_LAUNCHER_QSUB from ctsm.machine import CREATE_TEST_QUEUE_UNSPECIFIED from ctsm.machine_utils import get_user -MachineDefaults = namedtuple('MachineDefaults', ['job_launcher_type', - 'scratch_dir', - 'baseline_dir', - 'account_required', - 'create_test_retry', - 'create_test_queue', - 'job_launcher_defaults']) +MachineDefaults = namedtuple( + "MachineDefaults", + [ + "job_launcher_type", + "scratch_dir", + "baseline_dir", + "account_required", + "create_test_retry", + "create_test_queue", + "job_launcher_defaults", + ], +) # job_launcher_type: one of the JOB_LAUNCHERs defined in job_launcher_factory # scratch_dir: str # baseline_dir: str: The standard location for CTSM baselines on this machine @@ -37,51 +41,55 @@ # have defaults for qsub, because other launchers (like no_batch) don't need any # arguments. -QsubDefaults = namedtuple('QsubDefaults', ['queue', - 'walltime', - 'extra_args', - 'required_args']) +QsubDefaults = namedtuple( + "QsubDefaults", ["queue", "walltime", "extra_args", "required_args"] +) MACHINE_DEFAULTS = { - 'cheyenne': MachineDefaults( + "cheyenne": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir=os.path.join(os.path.sep, 'glade', 'scratch', get_user()), - baseline_dir=os.path.join(os.path.sep, 'glade', 'p', 'cgd', 'tss', 'ctsm_baselines'), + scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), + baseline_dir=os.path.join( + os.path.sep, "glade", "p", "cgd", "tss", "ctsm_baselines" + ), account_required=True, create_test_retry=0, # NOTE(wjs, 2022-02-23) By default, use the regular queue, even for # single-processor jobs. This is because the share queue has been really flaky, # with lots of job failures or slow-running jobs. - create_test_queue='regular', + create_test_queue="regular", job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='regular', - walltime='11:50:00', - extra_args='', + queue="regular", + walltime="11:50:00", + extra_args="", # The following assumes a single node, with a single mpi proc; we may want # to add more flexibility in the future, making the node / proc counts # individually selectable - required_args= - '-l select=1:ncpus=36:mpiprocs=1 -V -r n -l inception=login -k oed') - }), - 'hobart': MachineDefaults( + required_args="-l select=1:ncpus=36:mpiprocs=1 -V -r n -l inception=login -k oed", + ) + }, + ), + "hobart": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir=os.path.join(os.path.sep, 'scratch', 'cluster', get_user()), - baseline_dir=os.path.join(os.path.sep, 'fs', 'cgd', 'csm', 'ccsm_baselines'), + scratch_dir=os.path.join(os.path.sep, "scratch", "cluster", get_user()), + baseline_dir=os.path.join(os.path.sep, "fs", "cgd", "csm", "ccsm_baselines"), account_required=False, create_test_retry=0, create_test_queue=CREATE_TEST_QUEUE_UNSPECIFIED, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='medium', - walltime='04:00:00', - extra_args='', - required_args='-l nodes=1:ppn=48 -r n') - }), - 'izumi': MachineDefaults( + queue="medium", + walltime="04:00:00", + extra_args="", + required_args="-l nodes=1:ppn=48 -r n", + ) + }, + ), + "izumi": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir=os.path.join(os.path.sep, 'scratch', 'cluster', get_user()), - baseline_dir=os.path.join(os.path.sep, 'fs', 'cgd', 'csm', 'ccsm_baselines'), + scratch_dir=os.path.join(os.path.sep, "scratch", "cluster", get_user()), + baseline_dir=os.path.join(os.path.sep, "fs", "cgd", "csm", "ccsm_baselines"), account_required=False, # jobs on izumi experience a high frequency of failures, often at the very end of # the job; so we'll automatically retry a failed job twice before giving up on it @@ -89,9 +97,11 @@ create_test_queue=CREATE_TEST_QUEUE_UNSPECIFIED, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='medium', - walltime='04:00:00', - extra_args='', - required_args='-l nodes=1:ppn=48 -r n') - }) + queue="medium", + walltime="04:00:00", + extra_args="", + required_args="-l nodes=1:ppn=48 -r n", + ) + }, + ), } diff --git a/python/ctsm/machine_utils.py b/python/ctsm/machine_utils.py index b9cd9f8d84..da5c8b9c6a 100644 --- a/python/ctsm/machine_utils.py +++ b/python/ctsm/machine_utils.py @@ -11,23 +11,27 @@ # Public functions # ======================================================================== + def get_user(): """Return the current user name (string)""" return getpass.getuser() + def get_machine_name(): """Return the current machine name (string)""" # NOTE(wjs, 2021-12-13) The following line needs a "disable=no-member" to workaround a # problem on my Mac (probably similar to # https://stackoverflow.com/questions/68719442/why-do-i-get-the-pylint-error-module-socket-has-no-gethostname-member-no-m) - full_hostname = socket.gethostname() # pylint: disable=no-member - hostname = full_hostname.split('.')[0] + full_hostname = socket.gethostname() # pylint: disable=no-member + hostname = full_hostname.split(".")[0] return _machine_from_hostname(hostname) + # ======================================================================== # Private functions # ======================================================================== + def _machine_from_hostname(hostname): """Given a hostname (string), return the machine name (string) @@ -35,8 +39,8 @@ def _machine_from_hostname(hostname): be extended if there are other machines with special translation rules from hostname to machine name. """ - if re.match(r'cheyenne\d+', hostname): - machine = 'cheyenne' + if re.match(r"cheyenne\d+", hostname): + machine = "cheyenne" else: machine = hostname diff --git a/python/ctsm/modify_fsurdat/fsurdat_modifier.py b/python/ctsm/modify_fsurdat/fsurdat_modifier.py index f9c0ac9682..18e11cf2df 100644 --- a/python/ctsm/modify_fsurdat/fsurdat_modifier.py +++ b/python/ctsm/modify_fsurdat/fsurdat_modifier.py @@ -9,12 +9,17 @@ import argparse from configparser import ConfigParser from ctsm.config_utils import get_config_value -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) from ctsm.modify_fsurdat.modify_fsurdat import ModifyFsurdat logger = logging.getLogger(__name__) -def main (): + +def main(): """ Description ----------- @@ -26,13 +31,15 @@ def main (): # read the command line argument to obtain the path to the .cfg file parser = argparse.ArgumentParser() - parser.add_argument('cfg_path', - help='/path/name.cfg of input file, eg ./modify.cfg') + parser.add_argument( + "cfg_path", help="/path/name.cfg of input file, eg ./modify.cfg" + ) add_logging_args(parser) args = parser.parse_args() process_logging_args(args) fsurdat_modifier(args.cfg_path) + def fsurdat_modifier(cfg_path): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file @@ -41,66 +48,148 @@ def fsurdat_modifier(cfg_path): section = config.sections()[0] # name of the first section # required: user must set these in the .cfg file - fsurdat_in = get_config_value(config=config, section=section, - item='fsurdat_in', file_path=cfg_path) - fsurdat_out = get_config_value(config=config, section=section, - item='fsurdat_out', file_path=cfg_path) + fsurdat_in = get_config_value( + config=config, section=section, item="fsurdat_in", file_path=cfg_path + ) + fsurdat_out = get_config_value( + config=config, section=section, item="fsurdat_out", file_path=cfg_path + ) # required but fallback values available for variables omitted # entirely from the .cfg file - idealized = get_config_value(config=config, section=section, - item='idealized', file_path=cfg_path, convert_to_type=bool) - zero_nonveg = get_config_value(config=config, section=section, - item='zero_nonveg', file_path=cfg_path, convert_to_type=bool) - - lnd_lat_1 = get_config_value(config=config, section=section, - item='lnd_lat_1', file_path=cfg_path, convert_to_type=float) - lnd_lat_2 = get_config_value(config=config, section=section, - item='lnd_lat_2', file_path=cfg_path, convert_to_type=float) - lnd_lon_1 = get_config_value(config=config, section=section, - item='lnd_lon_1', file_path=cfg_path, convert_to_type=float) - lnd_lon_2 = get_config_value(config=config, section=section, - item='lnd_lon_2', file_path=cfg_path, convert_to_type=float) - - landmask_file = get_config_value(config=config, section=section, - item='landmask_file', file_path=cfg_path, can_be_unset=True) + idealized = get_config_value( + config=config, + section=section, + item="idealized", + file_path=cfg_path, + convert_to_type=bool, + ) + zero_nonveg = get_config_value( + config=config, + section=section, + item="zero_nonveg", + file_path=cfg_path, + convert_to_type=bool, + ) + + lnd_lat_1 = get_config_value( + config=config, + section=section, + item="lnd_lat_1", + file_path=cfg_path, + convert_to_type=float, + ) + lnd_lat_2 = get_config_value( + config=config, + section=section, + item="lnd_lat_2", + file_path=cfg_path, + convert_to_type=float, + ) + lnd_lon_1 = get_config_value( + config=config, + section=section, + item="lnd_lon_1", + file_path=cfg_path, + convert_to_type=float, + ) + lnd_lon_2 = get_config_value( + config=config, + section=section, + item="lnd_lon_2", + file_path=cfg_path, + convert_to_type=float, + ) + + landmask_file = get_config_value( + config=config, + section=section, + item="landmask_file", + file_path=cfg_path, + can_be_unset=True, + ) # Create ModifyFsurdat object - modify_fsurdat = ModifyFsurdat.init_from_file(fsurdat_in, - lnd_lon_1, lnd_lon_2, lnd_lat_1, lnd_lat_2, landmask_file) + modify_fsurdat = ModifyFsurdat.init_from_file( + fsurdat_in, lnd_lon_1, lnd_lon_2, lnd_lat_1, lnd_lat_2, landmask_file + ) # not required: user may set these in the .cfg file max_pft = int(max(modify_fsurdat.file.lsmpft)) - dom_plant = get_config_value(config=config, section=section, - item='dom_plant', file_path=cfg_path, + dom_plant = get_config_value( + config=config, + section=section, + item="dom_plant", + file_path=cfg_path, allowed_values=range(max_pft + 1), # integers from 0 to max_pft - convert_to_type=int, can_be_unset=True) - - lai = get_config_value(config=config, section=section, item='lai', - file_path=cfg_path, is_list=True, - convert_to_type=float, can_be_unset=True) - sai = get_config_value(config=config, section=section, item='sai', - file_path=cfg_path, is_list=True, - convert_to_type=float, can_be_unset=True) - hgt_top = get_config_value(config=config, section=section, - item='hgt_top', file_path=cfg_path, is_list=True, - convert_to_type=float, can_be_unset=True) - hgt_bot = get_config_value(config=config, section=section, - item='hgt_bot', file_path=cfg_path, is_list=True, - convert_to_type=float, can_be_unset=True) + convert_to_type=int, + can_be_unset=True, + ) + + lai = get_config_value( + config=config, + section=section, + item="lai", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + sai = get_config_value( + config=config, + section=section, + item="sai", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + hgt_top = get_config_value( + config=config, + section=section, + item="hgt_top", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + hgt_bot = get_config_value( + config=config, + section=section, + item="hgt_bot", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) max_soil_color = int(modify_fsurdat.file.mxsoil_color) - soil_color = get_config_value(config=config, section=section, - item='soil_color', file_path=cfg_path, + soil_color = get_config_value( + config=config, + section=section, + item="soil_color", + file_path=cfg_path, allowed_values=range(1, max_soil_color + 1), # 1 to max_soil_color - convert_to_type=int, can_be_unset=True) - - std_elev = get_config_value(config=config, section=section, - item='std_elev', file_path=cfg_path, - convert_to_type=float, can_be_unset=True) - max_sat_area = get_config_value(config=config, section=section, - item='max_sat_area', file_path=cfg_path, - convert_to_type=float, can_be_unset=True) + convert_to_type=int, + can_be_unset=True, + ) + + std_elev = get_config_value( + config=config, + section=section, + item="std_elev", + file_path=cfg_path, + convert_to_type=float, + can_be_unset=True, + ) + max_sat_area = get_config_value( + config=config, + section=section, + item="max_sat_area", + file_path=cfg_path, + convert_to_type=float, + can_be_unset=True, + ) # ------------------------------ # modify surface data properties @@ -114,33 +203,34 @@ def fsurdat_modifier(cfg_path): if idealized: modify_fsurdat.set_idealized() # set 2D variables # set 3D and 4D variables pertaining to natural vegetation - modify_fsurdat.set_dom_plant(dom_plant=0, lai=[], sai=[], - hgt_top=[], hgt_bot=[]) - logger.info('idealized complete') + modify_fsurdat.set_dom_plant( + dom_plant=0, lai=[], sai=[], hgt_top=[], hgt_bot=[] + ) + logger.info("idealized complete") if max_sat_area is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0('FMAX', max_sat_area) - logger.info('max_sat_area complete') + modify_fsurdat.setvar_lev0("FMAX", max_sat_area) + logger.info("max_sat_area complete") if std_elev is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0('STD_ELEV', std_elev) - logger.info('std_elev complete') + modify_fsurdat.setvar_lev0("STD_ELEV", std_elev) + logger.info("std_elev complete") if soil_color is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0('SOIL_COLOR', soil_color) - logger.info('soil_color complete') + modify_fsurdat.setvar_lev0("SOIL_COLOR", soil_color) + logger.info("soil_color complete") if zero_nonveg: modify_fsurdat.zero_nonveg() - logger.info('zero_nonveg complete') + logger.info("zero_nonveg complete") # The set_dom_plant call follows zero_nonveg because it modifies PCT_NATVEG # and PCT_CROP in the user-defined rectangle if dom_plant is not None: - modify_fsurdat.set_dom_plant(dom_plant=dom_plant, - lai=lai, sai=sai, - hgt_top=hgt_top, hgt_bot=hgt_bot) - logger.info('dom_plant complete') + modify_fsurdat.set_dom_plant( + dom_plant=dom_plant, lai=lai, sai=sai, hgt_top=hgt_top, hgt_bot=hgt_bot + ) + logger.info("dom_plant complete") # ---------------------------------------------- # Output the now modified CTSM surface data file diff --git a/python/ctsm/modify_fsurdat/modify_fsurdat.py b/python/ctsm/modify_fsurdat/modify_fsurdat.py index 10803875c7..3e7b267505 100644 --- a/python/ctsm/modify_fsurdat/modify_fsurdat.py +++ b/python/ctsm/modify_fsurdat/modify_fsurdat.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) + class ModifyFsurdat: """ Description @@ -28,9 +29,13 @@ def __init__(self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file): self.file = my_data self.rectangle = self._get_rectangle( - lon_1=lon_1, lon_2=lon_2, - lat_1=lat_1, lat_2=lat_2, - longxy=self.file.LONGXY, latixy=self.file.LATIXY) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=self.file.LONGXY, + latixy=self.file.LATIXY, + ) if landmask_file is not None: # overwrite self.not_rectangle with data from @@ -40,15 +45,13 @@ def __init__(self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file): self.not_rectangle = np.logical_not(self.rectangle) - @classmethod def init_from_file(cls, fsurdat_in, lon_1, lon_2, lat_1, lat_2, landmask_file): """Initialize a ModifyFsurdat object from file fsurdat_in""" - logger.info('Opening fsurdat_in file to be modified: %s', fsurdat_in) + logger.info("Opening fsurdat_in file to be modified: %s", fsurdat_in) my_file = xr.open_dataset(fsurdat_in) return cls(my_file, lon_1, lon_2, lat_1, lat_2, landmask_file) - @staticmethod def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy): """ @@ -62,11 +65,11 @@ def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy): # determine the rectangle(s) # TODO This is not really "nearest" for the edges but isel didn't work - rectangle_1 = (longxy >= lon_1) - rectangle_2 = (longxy <= lon_2) + rectangle_1 = longxy >= lon_1 + rectangle_2 = longxy <= lon_2 eps = np.finfo(np.float32).eps # to avoid roundoff issue - rectangle_3 = (latixy >= (lat_1 - eps)) - rectangle_4 = (latixy <= (lat_2 + eps)) + rectangle_3 = latixy >= (lat_1 - eps) + rectangle_4 = latixy <= (lat_2 + eps) if lon_1 <= lon_2: # rectangles overlap @@ -76,7 +79,7 @@ def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy): union_1 = np.logical_or(rectangle_1, rectangle_2) if lat_1 < -90 or lat_1 > 90 or lat_2 < -90 or lat_2 > 90: - errmsg = 'lat_1 and lat_2 need to be in the range -90 to 90' + errmsg = "lat_1 and lat_2 need to be in the range -90 to 90" abort(errmsg) elif lat_1 <= lat_2: # rectangles overlap @@ -90,7 +93,6 @@ def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy): return rectangle - def write_output(self, fsurdat_in, fsurdat_out): """ Description @@ -107,28 +109,31 @@ def write_output(self, fsurdat_in, fsurdat_out): # update attributes # TODO Better as dictionary? - title = 'Modified fsurdat file' - summary = 'Modified fsurdat file' - contact = 'N/A' + title = "Modified fsurdat file" + summary = "Modified fsurdat file" + contact = "N/A" data_script = os.path.abspath(__file__) + " -- " + get_ctsm_git_short_hash() - description = 'Modified this file: ' + fsurdat_in - update_metadata(self.file, title=title, summary=summary, - contact=contact, data_script=data_script, - description=description) + description = "Modified this file: " + fsurdat_in + update_metadata( + self.file, + title=title, + summary=summary, + contact=contact, + data_script=data_script, + description=description, + ) # abort if output file already exists file_exists = os.path.exists(fsurdat_out) if file_exists: - errmsg = 'Output file already exists: ' + fsurdat_out + errmsg = "Output file already exists: " + fsurdat_out abort(errmsg) # mode 'w' overwrites file if it exists - self.file.to_netcdf(path=fsurdat_out, mode='w', - format="NETCDF3_64BIT") - logger.info('Successfully created fsurdat_out: %s', fsurdat_out) + self.file.to_netcdf(path=fsurdat_out, mode="w", format="NETCDF3_64BIT") + logger.info("Successfully created fsurdat_out: %s", fsurdat_out) self.file.close() - def set_dom_plant(self, dom_plant, lai, sai, hgt_top, hgt_bot): """ Description @@ -157,35 +162,38 @@ def set_dom_plant(self, dom_plant, lai, sai, hgt_top, hgt_bot): # If dom_plant is a cft, add PCT_NATVEG to PCT_CROP in the rectangle # and remove same from PCT_NATVEG, i.e. set PCT_NATVEG = 0. if dom_plant > max(self.file.natpft): # dom_plant is a cft (crop) - self.file['PCT_CROP'] = \ - self.file['PCT_CROP'] + \ - self.file['PCT_NATVEG'].where(self.rectangle, other=0) - self.setvar_lev0('PCT_NATVEG', 0) + self.file["PCT_CROP"] = self.file["PCT_CROP"] + self.file[ + "PCT_NATVEG" + ].where(self.rectangle, other=0) + self.setvar_lev0("PCT_NATVEG", 0) for cft in self.file.cft: cft_local = cft - (max(self.file.natpft) + 1) # initialize 3D variable; set outside the loop below - self.setvar_lev1('PCT_CFT', val=0, lev1_dim=cft_local) + self.setvar_lev1("PCT_CFT", val=0, lev1_dim=cft_local) # set 3D variable - self.setvar_lev1('PCT_CFT', val=100, lev1_dim=dom_plant-(max(self.file.natpft)+1)) + self.setvar_lev1( + "PCT_CFT", val=100, lev1_dim=dom_plant - (max(self.file.natpft) + 1) + ) else: # dom_plant is a pft (not a crop) for pft in self.file.natpft: # initialize 3D variable; set outside the loop below - self.setvar_lev1('PCT_NAT_PFT', val=0, lev1_dim=pft) + self.setvar_lev1("PCT_NAT_PFT", val=0, lev1_dim=pft) # set 3D variable value for dom_plant - self.setvar_lev1('PCT_NAT_PFT', val=100, lev1_dim=dom_plant) + self.setvar_lev1("PCT_NAT_PFT", val=100, lev1_dim=dom_plant) # dictionary of 4d variables to loop over - vars_4d = {'MONTHLY_LAI': lai, - 'MONTHLY_SAI': sai, - 'MONTHLY_HEIGHT_TOP': hgt_top, - 'MONTHLY_HEIGHT_BOT': hgt_bot} + vars_4d = { + "MONTHLY_LAI": lai, + "MONTHLY_SAI": sai, + "MONTHLY_HEIGHT_TOP": hgt_top, + "MONTHLY_HEIGHT_BOT": hgt_bot, + } for var, val in vars_4d.items(): if val is not None: self.set_lai_sai_hgts(dom_plant=dom_plant, var=var, val=val) - def set_lai_sai_hgts(self, dom_plant, var, val): """ Description @@ -197,14 +205,16 @@ def set_lai_sai_hgts(self, dom_plant, var, val): if dom_plant == 0: # bare soil: var must equal 0 val = [0] * months if len(val) != months: - errmsg = 'Error: Variable should have exactly ' + months + \ - ' entries in the configure file: ' + var + errmsg = ( + "Error: Variable should have exactly " + + months + + " entries in the configure file: " + + var + ) abort(errmsg) for mon in self.file.time - 1: # loop over 12 months # set 4D variable to value for dom_plant - self.setvar_lev2(var, val[int(mon)], lev1_dim=dom_plant, - lev2_dim=mon) - + self.setvar_lev2(var, val[int(mon)], lev1_dim=dom_plant, lev2_dim=mon) def zero_nonveg(self): """ @@ -214,22 +224,19 @@ def zero_nonveg(self): Set that one to 100%. """ - self.setvar_lev0('PCT_NATVEG', 100) - self.setvar_lev0('PCT_CROP', 0) - self.setvar_lev0('PCT_LAKE', 0) - self.setvar_lev0('PCT_WETLAND', 0) - self.setvar_lev0('PCT_URBAN', 0) - self.setvar_lev0('PCT_GLACIER', 0) - + self.setvar_lev0("PCT_NATVEG", 100) + self.setvar_lev0("PCT_CROP", 0) + self.setvar_lev0("PCT_LAKE", 0) + self.setvar_lev0("PCT_WETLAND", 0) + self.setvar_lev0("PCT_URBAN", 0) + self.setvar_lev0("PCT_GLACIER", 0) def setvar_lev0(self, var, val): """ Sets 2d variable var to value val in user-defined rectangle, defined as "other" in the function """ - self.file[var] = self.file[var].where( - self.not_rectangle, other=val) - + self.file[var] = self.file[var].where(self.not_rectangle, other=val) def setvar_lev1(self, var, val, lev1_dim): """ @@ -237,18 +244,17 @@ def setvar_lev1(self, var, val, lev1_dim): defined as "other" in the function """ self.file[var][lev1_dim, ...] = self.file[var][lev1_dim, ...].where( - self.not_rectangle, other=val) - + self.not_rectangle, other=val + ) def setvar_lev2(self, var, val, lev1_dim, lev2_dim): """ Sets 4d variable var to value val in user-defined rectangle, defined as "other" in the function """ - self.file[var][lev2_dim,lev1_dim, ...] = \ - self.file[var][lev2_dim,lev1_dim, ...].where( - self.not_rectangle, other=val) - + self.file[var][lev2_dim, lev1_dim, ...] = self.file[var][ + lev2_dim, lev1_dim, ... + ].where(self.not_rectangle, other=val) def set_idealized(self): """ @@ -282,31 +288,31 @@ def set_idealized(self): organic = 0 # 2D variables - self.setvar_lev0('FMAX', max_sat_area) - self.setvar_lev0('STD_ELEV', std_elev) - self.setvar_lev0('SLOPE', slope) - self.setvar_lev0('zbedrock', zbedrock) - self.setvar_lev0('SOIL_COLOR', soil_color) - self.setvar_lev0('PFTDATA_MASK', pftdata_mask) - self.setvar_lev0('LANDFRAC_PFT', landfrac_pft) - self.setvar_lev0('PCT_WETLAND', pct_not_nat_veg) - self.setvar_lev0('PCT_CROP', pct_not_nat_veg) - self.setvar_lev0('PCT_LAKE', pct_not_nat_veg) - self.setvar_lev0('PCT_URBAN', pct_not_nat_veg) - self.setvar_lev0('PCT_GLACIER', pct_not_nat_veg) - self.setvar_lev0('PCT_NATVEG', pct_nat_veg) + self.setvar_lev0("FMAX", max_sat_area) + self.setvar_lev0("STD_ELEV", std_elev) + self.setvar_lev0("SLOPE", slope) + self.setvar_lev0("zbedrock", zbedrock) + self.setvar_lev0("SOIL_COLOR", soil_color) + self.setvar_lev0("PFTDATA_MASK", pftdata_mask) + self.setvar_lev0("LANDFRAC_PFT", landfrac_pft) + self.setvar_lev0("PCT_WETLAND", pct_not_nat_veg) + self.setvar_lev0("PCT_CROP", pct_not_nat_veg) + self.setvar_lev0("PCT_LAKE", pct_not_nat_veg) + self.setvar_lev0("PCT_URBAN", pct_not_nat_veg) + self.setvar_lev0("PCT_GLACIER", pct_not_nat_veg) + self.setvar_lev0("PCT_NATVEG", pct_nat_veg) for lev in self.file.nlevsoi: # set next three 3D variables to values representing loam - self.setvar_lev1('PCT_SAND', val=pct_sand, lev1_dim=lev) - self.setvar_lev1('PCT_CLAY', val=pct_clay, lev1_dim=lev) - self.setvar_lev1('ORGANIC', val=organic, lev1_dim=lev) + self.setvar_lev1("PCT_SAND", val=pct_sand, lev1_dim=lev) + self.setvar_lev1("PCT_CLAY", val=pct_clay, lev1_dim=lev) + self.setvar_lev1("ORGANIC", val=organic, lev1_dim=lev) for crop in self.file.cft: cft_local = crop - (max(self.file.natpft) + 1) # initialize 3D variable; set outside the loop below - self.setvar_lev1('PCT_CFT', val=0, lev1_dim=cft_local) + self.setvar_lev1("PCT_CFT", val=0, lev1_dim=cft_local) # set 3D variable # NB. sum(PCT_CFT) must = 100 even though PCT_CROP = 0 - self.setvar_lev1('PCT_CFT', val=100, lev1_dim=0) + self.setvar_lev1("PCT_CFT", val=100, lev1_dim=0) diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py index a72ac59a98..54e85ca85d 100644 --- a/python/ctsm/run_sys_tests.py +++ b/python/ctsm/run_sys_tests.py @@ -10,10 +10,17 @@ from CIME.test_utils import get_tests_from_xml # pylint: disable=import-error from CIME.cs_status_creator import create_cs_status # pylint: disable=import-error -from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args +from ctsm.ctsm_logging import ( + setup_logging_pre_config, + add_logging_args, + process_logging_args, +) from ctsm.machine_utils import get_machine_name -from ctsm.machine import (create_machine, get_possibly_overridden_mach_value, - CREATE_TEST_QUEUE_UNSPECIFIED) +from ctsm.machine import ( + create_machine, + get_possibly_overridden_mach_value, + CREATE_TEST_QUEUE_UNSPECIFIED, +) from ctsm.machine_defaults import MACHINE_DEFAULTS from ctsm.os_utils import make_link from ctsm.path_utils import path_to_ctsm_root @@ -28,12 +35,13 @@ _NICE_LEVEL = 19 # Extra arguments for the cs.status.fails command -_CS_STATUS_FAILS_EXTRA_ARGS = '--fails-only --count-performance-fails' +_CS_STATUS_FAILS_EXTRA_ARGS = "--fails-only --count-performance-fails" # ======================================================================== # Public functions # ======================================================================== + def main(cime_path): """Main function called when run_sys_tests is run from the command-line @@ -46,48 +54,67 @@ def main(cime_path): setup_logging_pre_config() args = _commandline_args() process_logging_args(args) - logger.info('Running on machine: %s', args.machine_name) + logger.info("Running on machine: %s", args.machine_name) if args.job_launcher_nobatch: job_launcher_type = JOB_LAUNCHER_NOBATCH else: job_launcher_type = None - machine = create_machine(machine_name=args.machine_name, - job_launcher_type=job_launcher_type, - defaults=MACHINE_DEFAULTS, - account=args.account, - job_launcher_queue=args.job_launcher_queue, - job_launcher_walltime=args.job_launcher_walltime, - job_launcher_nice_level=_NICE_LEVEL, - job_launcher_extra_args=args.job_launcher_extra_args) + machine = create_machine( + machine_name=args.machine_name, + job_launcher_type=job_launcher_type, + defaults=MACHINE_DEFAULTS, + account=args.account, + job_launcher_queue=args.job_launcher_queue, + job_launcher_walltime=args.job_launcher_walltime, + job_launcher_nice_level=_NICE_LEVEL, + job_launcher_extra_args=args.job_launcher_extra_args, + ) logger.debug("Machine info: %s", machine) - run_sys_tests(machine=machine, cime_path=cime_path, - skip_testroot_creation=args.skip_testroot_creation, - skip_git_status=args.skip_git_status, - dry_run=args.dry_run, - suite_name=args.suite_name, testfile=args.testfile, testlist=args.testname, - suite_compilers=args.suite_compiler, - testid_base=args.testid_base, testroot_base=args.testroot_base, - rerun_existing_failures=args.rerun_existing_failures, - compare_name=args.compare, generate_name=args.generate, - baseline_root=args.baseline_root, - walltime=args.walltime, queue=args.queue, - retry=args.retry, - extra_create_test_args=args.extra_create_test_args) - -def run_sys_tests(machine, cime_path, - skip_testroot_creation=False, - skip_git_status=False, - dry_run=False, - suite_name=None, testfile=None, testlist=None, - suite_compilers=None, - testid_base=None, testroot_base=None, - rerun_existing_failures=False, - compare_name=None, generate_name=None, - baseline_root=None, - walltime=None, queue=None, - retry=None, - extra_create_test_args=''): + run_sys_tests( + machine=machine, + cime_path=cime_path, + skip_testroot_creation=args.skip_testroot_creation, + skip_git_status=args.skip_git_status, + dry_run=args.dry_run, + suite_name=args.suite_name, + testfile=args.testfile, + testlist=args.testname, + suite_compilers=args.suite_compiler, + testid_base=args.testid_base, + testroot_base=args.testroot_base, + rerun_existing_failures=args.rerun_existing_failures, + compare_name=args.compare, + generate_name=args.generate, + baseline_root=args.baseline_root, + walltime=args.walltime, + queue=args.queue, + retry=args.retry, + extra_create_test_args=args.extra_create_test_args, + ) + + +def run_sys_tests( + machine, + cime_path, + skip_testroot_creation=False, + skip_git_status=False, + dry_run=False, + suite_name=None, + testfile=None, + testlist=None, + suite_compilers=None, + testid_base=None, + testroot_base=None, + rerun_existing_failures=False, + compare_name=None, + generate_name=None, + baseline_root=None, + walltime=None, + queue=None, + retry=None, + extra_create_test_args="", +): """Implementation of run_sys_tests command Exactly one of suite_name, testfile or testlist should be provided @@ -129,11 +156,15 @@ def run_sys_tests(machine, cime_path, testlist: list of strings giving test names to run """ - num_provided_options = ((suite_name is not None) + - (testfile is not None) + - (testlist is not None and len(testlist) > 0)) + num_provided_options = ( + (suite_name is not None) + + (testfile is not None) + + (testlist is not None and len(testlist) > 0) + ) if num_provided_options != 1: - raise RuntimeError("Exactly one of suite_name, testfile or testlist must be provided") + raise RuntimeError( + "Exactly one of suite_name, testfile or testlist must be provided" + ) if testid_base is None: testid_base = _get_testid_base(machine.name) @@ -144,12 +175,15 @@ def run_sys_tests(machine, cime_path, _make_testroot(testroot, testid_base, dry_run) else: if not os.path.exists(testroot): - raise RuntimeError("testroot directory does NOT exist as expected when a rerun" + - " option is used: directory expected = "+testroot ) + raise RuntimeError( + "testroot directory does NOT exist as expected when a rerun" + + " option is used: directory expected = " + + testroot + ) print("Testroot: {}\n".format(testroot)) - retry_final = get_possibly_overridden_mach_value(machine, - varname='create_test_retry', - value=retry) + retry_final = get_possibly_overridden_mach_value( + machine, varname="create_test_retry", value=retry + ) # Note the distinction between a queue of None and a queue of # CREATE_TEST_QUEUE_UNSPECIFIED in the following: If queue is None (meaning that the # user hasn't specified a '--queue' argument to run_sys_tests), then we'll use the @@ -159,58 +193,68 @@ def run_sys_tests(machine, cime_path, # (It's also possible for the machine object to specify a queue of # CREATE_TEST_QUEUE_UNSPECIFIED, which means that we won't use a '--queue' argument to # create_test unless the user specifies a '--queue' argument to run_sys_tests.) - queue_final = get_possibly_overridden_mach_value(machine, - varname='create_test_queue', - value=queue) + queue_final = get_possibly_overridden_mach_value( + machine, varname="create_test_queue", value=queue + ) if queue_final == CREATE_TEST_QUEUE_UNSPECIFIED: queue_final = None if not skip_git_status: _record_git_status(testroot, retry_final, dry_run) - baseline_root_final = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value=baseline_root) - create_test_args = _get_create_test_args(compare_name=compare_name, - generate_name=generate_name, - baseline_root=baseline_root_final, - account=machine.account, - walltime=walltime, - queue=queue_final, - retry=retry_final, - rerun_existing_failures=rerun_existing_failures, - extra_create_test_args=extra_create_test_args) + baseline_root_final = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value=baseline_root + ) + create_test_args = _get_create_test_args( + compare_name=compare_name, + generate_name=generate_name, + baseline_root=baseline_root_final, + account=machine.account, + walltime=walltime, + queue=queue_final, + retry=retry_final, + rerun_existing_failures=rerun_existing_failures, + extra_create_test_args=extra_create_test_args, + ) if suite_name: if not dry_run: _make_cs_status_for_suite(testroot, testid_base) - _run_test_suite(cime_path=cime_path, - suite_name=suite_name, - suite_compilers=suite_compilers, - machine=machine, - testid_base=testid_base, testroot=testroot, - create_test_args=create_test_args, - dry_run=dry_run) + _run_test_suite( + cime_path=cime_path, + suite_name=suite_name, + suite_compilers=suite_compilers, + machine=machine, + testid_base=testid_base, + testroot=testroot, + create_test_args=create_test_args, + dry_run=dry_run, + ) else: if not dry_run: _make_cs_status_non_suite(testroot, testid_base) if testfile: - test_args = ['--testfile', os.path.abspath(testfile)] + test_args = ["--testfile", os.path.abspath(testfile)] elif testlist: test_args = testlist else: raise RuntimeError("None of suite_name, testfile or testlist were provided") - _run_create_test(cime_path=cime_path, - test_args=test_args, machine=machine, - testid=testid_base, testroot=testroot, - create_test_args=create_test_args, - dry_run=dry_run) + _run_create_test( + cime_path=cime_path, + test_args=test_args, + machine=machine, + testid=testid_base, + testroot=testroot, + create_test_args=create_test_args, + dry_run=dry_run, + ) + # ======================================================================== # Private functions # ======================================================================== + def _commandline_args(): - """Parse and return command-line arguments - """ + """Parse and return command-line arguments""" description = """ Driver for running CTSM system tests @@ -236,157 +280,220 @@ def _commandline_args(): """ parser = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter) + description=description, formatter_class=argparse.RawTextHelpFormatter + ) machine_name = get_machine_name() - default_machine = create_machine(machine_name, - defaults=MACHINE_DEFAULTS, - allow_missing_entries=True) + default_machine = create_machine( + machine_name, defaults=MACHINE_DEFAULTS, allow_missing_entries=True + ) tests_to_run = parser.add_mutually_exclusive_group(required=True) - tests_to_run.add_argument('-s', '--suite-name', - help='Name of test suite to run') + tests_to_run.add_argument("-s", "--suite-name", help="Name of test suite to run") - tests_to_run.add_argument('-f', '--testfile', - help='Path to file listing tests to run') + tests_to_run.add_argument( + "-f", "--testfile", help="Path to file listing tests to run" + ) - tests_to_run.add_argument('-t', '--testname', '--testnames', nargs='+', - help='One or more test names to run (space-delimited)') + tests_to_run.add_argument( + "-t", + "--testname", + "--testnames", + nargs="+", + help="One or more test names to run (space-delimited)", + ) compare = parser.add_mutually_exclusive_group(required=True) - compare.add_argument('-c', '--compare', metavar='COMPARE_NAME', - help='Baseline name (often tag) to compare against\n' - '(required unless --skip-compare is given)') + compare.add_argument( + "-c", + "--compare", + metavar="COMPARE_NAME", + help="Baseline name (often tag) to compare against\n" + "(required unless --skip-compare is given)", + ) - compare.add_argument('--skip-compare', action='store_true', - help='Do not compare against baselines') + compare.add_argument( + "--skip-compare", action="store_true", help="Do not compare against baselines" + ) generate = parser.add_mutually_exclusive_group(required=True) - generate.add_argument('-g', '--generate', metavar='GENERATE_NAME', - help='Baseline name (often tag) to generate\n' - '(required unless --skip-generate is given)') - - generate.add_argument('--skip-generate', action='store_true', - help='Do not generate baselines') - - parser.add_argument('--suite-compiler', '--suite-compilers', nargs='+', - help='Compiler(s) from the given test suite for which tests are run\n' - 'Only valid in conjunction with -s/--suite-name;\n' - 'if not specified, use all compilers defined for this suite and machine\n') - - parser.add_argument('--account', - help='Account number to use for job submission.\n' - 'This is needed on some machines; if not provided explicitly,\n' - 'the script will attempt to guess it using the same rules as in CIME.\n' - 'Default for this machine: {}'.format(default_machine.account)) - - parser.add_argument('--testid-base', - help='Base string used for the test id.\n' - 'Default is to auto-generate this with a date and time stamp.') - - parser.add_argument('--testroot-base', - help='Path in which testroot should be put.\n' - 'For supported machines, this can be left out;\n' - 'for non-supported machines, it must be provided.\n' - 'Default for this machine: {}'.format(default_machine.scratch_dir)) - - parser.add_argument('--rerun-existing-failures', action='store_true', - help='Rerun failed tests from the last PEND or FAIL state.\n' - 'This triggers the --use-existing option to create_test.\n' - 'To use this option, provide the same options to run_sys_tests\n' - 'as in the initial run, but also adding --testid-base\n' - 'corresponding to the base testid used initially.\n' - '(However, many of the arguments to create_test are ignored,\n' - 'so it is not important for all of the options to exactly match\n' - 'those in the initial run.)\n' - 'This option implies --skip-testroot-creation (that option does not\n' - 'need to be specified separately if using --rerun-existing-failures).') + generate.add_argument( + "-g", + "--generate", + metavar="GENERATE_NAME", + help="Baseline name (often tag) to generate\n" + "(required unless --skip-generate is given)", + ) + + generate.add_argument( + "--skip-generate", action="store_true", help="Do not generate baselines" + ) + + parser.add_argument( + "--suite-compiler", + "--suite-compilers", + nargs="+", + help="Compiler(s) from the given test suite for which tests are run\n" + "Only valid in conjunction with -s/--suite-name;\n" + "if not specified, use all compilers defined for this suite and machine\n", + ) + + parser.add_argument( + "--account", + help="Account number to use for job submission.\n" + "This is needed on some machines; if not provided explicitly,\n" + "the script will attempt to guess it using the same rules as in CIME.\n" + "Default for this machine: {}".format(default_machine.account), + ) + + parser.add_argument( + "--testid-base", + help="Base string used for the test id.\n" + "Default is to auto-generate this with a date and time stamp.", + ) + + parser.add_argument( + "--testroot-base", + help="Path in which testroot should be put.\n" + "For supported machines, this can be left out;\n" + "for non-supported machines, it must be provided.\n" + "Default for this machine: {}".format(default_machine.scratch_dir), + ) + + parser.add_argument( + "--rerun-existing-failures", + action="store_true", + help="Rerun failed tests from the last PEND or FAIL state.\n" + "This triggers the --use-existing option to create_test.\n" + "To use this option, provide the same options to run_sys_tests\n" + "as in the initial run, but also adding --testid-base\n" + "corresponding to the base testid used initially.\n" + "(However, many of the arguments to create_test are ignored,\n" + "so it is not important for all of the options to exactly match\n" + "those in the initial run.)\n" + "This option implies --skip-testroot-creation (that option does not\n" + "need to be specified separately if using --rerun-existing-failures).", + ) if default_machine.baseline_dir: - baseline_root_default_msg = 'Default for this machine: {}'.format( - default_machine.baseline_dir) + baseline_root_default_msg = "Default for this machine: {}".format( + default_machine.baseline_dir + ) else: baseline_root_default_msg = "Default for this machine: use cime's default" - parser.add_argument('--baseline-root', - help='Path in which baselines should be compared and generated.\n' + - baseline_root_default_msg) - - parser.add_argument('--walltime', - help='Walltime for each test.\n' - 'If running a test suite, you can generally leave this unset,\n' - 'because it is set in the file defining the test suite.\n' - 'For other uses, providing this helps decrease the time spent\n' - 'waiting in the queue.') - - parser.add_argument('--queue', - help='Queue to which tests are submitted.\n' - 'The special value "{}" means do not add a --queue option to create_test,\n' - 'instead allowing CIME to pick an appropriate queue for each test\n' - 'using its standard mechanisms.\n' - 'Default for this machine: {}'.format( - CREATE_TEST_QUEUE_UNSPECIFIED, default_machine.create_test_queue)) - - parser.add_argument('--retry', type=int, - help='Argument to create_test: Number of times to retry failed tests.\n' - 'Default for this machine: {}'.format( - default_machine.create_test_retry)) - - parser.add_argument('--extra-create-test-args', default='', - help='String giving extra arguments to pass to create_test\n' - '(To allow the argument parsing to accept this, enclose the string\n' - 'in quotes, with a leading space, as in " --my-arg foo".)') - - parser.add_argument('--job-launcher-nobatch', action='store_true', - help='Run create_test on the login node, even if this machine\n' - 'is set up to submit create_test to a compute node by default.') - - parser.add_argument('--job-launcher-queue', - help='Queue to which the create_test command is submitted.\n' - 'Only applies on machines for which we submit the create_test command\n' - 'rather than running it on the login node.\n' - 'Default for this machine: {}'.format( - default_machine.job_launcher.get_queue())) - - parser.add_argument('--job-launcher-walltime', - help='Walltime for the create_test command.\n' - 'Only applies on machines for which we submit the create_test command\n' - 'rather than running it on the login node.\n' - 'Default for this machine: {}'.format( - default_machine.job_launcher.get_walltime())) - - parser.add_argument('--job-launcher-extra-args', - help='Extra arguments for the command that launches the\n' - 'create_test command.\n' - '(To allow the argument parsing to accept this, enclose the string\n' - 'in quotes, with a leading space, as in " --my-arg foo".)\n' - 'Default for this machine: {}'.format( - default_machine.job_launcher.get_extra_args())) - - parser.add_argument('--skip-testroot-creation', action='store_true', - help='Do not create the directory that will hold the tests.\n' - 'This should be used if the desired testroot directory already exists.') - - parser.add_argument('--skip-git-status', action='store_true', - help='Skip printing git and manage_externals status,\n' - 'both to screen and to the SRCROOT_GIT_STATUS file in TESTROOT.\n' - 'This printing can often be helpful, but this option can be used to\n' - 'avoid extraneous output, to reduce the time needed to run this script,\n' - 'or if git or manage_externals are currently broken in your sandbox.\n') - - parser.add_argument('--dry-run', action='store_true', - help='Print what would happen, but do not run any commands.\n' - '(Generally should be run with --verbose.)\n') - - parser.add_argument('--machine-name', default=machine_name, - help='Name of machine for which create_test is run.\n' - 'This typically is not needed, but can be provided\n' - 'for the sake of testing this script.\n' - 'Defaults to current machine: {}'.format(machine_name)) + parser.add_argument( + "--baseline-root", + help="Path in which baselines should be compared and generated.\n" + + baseline_root_default_msg, + ) + + parser.add_argument( + "--walltime", + help="Walltime for each test.\n" + "If running a test suite, you can generally leave this unset,\n" + "because it is set in the file defining the test suite.\n" + "For other uses, providing this helps decrease the time spent\n" + "waiting in the queue.", + ) + + parser.add_argument( + "--queue", + help="Queue to which tests are submitted.\n" + 'The special value "{}" means do not add a --queue option to create_test,\n' + "instead allowing CIME to pick an appropriate queue for each test\n" + "using its standard mechanisms.\n" + "Default for this machine: {}".format( + CREATE_TEST_QUEUE_UNSPECIFIED, default_machine.create_test_queue + ), + ) + + parser.add_argument( + "--retry", + type=int, + help="Argument to create_test: Number of times to retry failed tests.\n" + "Default for this machine: {}".format(default_machine.create_test_retry), + ) + + parser.add_argument( + "--extra-create-test-args", + default="", + help="String giving extra arguments to pass to create_test\n" + "(To allow the argument parsing to accept this, enclose the string\n" + 'in quotes, with a leading space, as in " --my-arg foo".)', + ) + + parser.add_argument( + "--job-launcher-nobatch", + action="store_true", + help="Run create_test on the login node, even if this machine\n" + "is set up to submit create_test to a compute node by default.", + ) + + parser.add_argument( + "--job-launcher-queue", + help="Queue to which the create_test command is submitted.\n" + "Only applies on machines for which we submit the create_test command\n" + "rather than running it on the login node.\n" + "Default for this machine: {}".format(default_machine.job_launcher.get_queue()), + ) + + parser.add_argument( + "--job-launcher-walltime", + help="Walltime for the create_test command.\n" + "Only applies on machines for which we submit the create_test command\n" + "rather than running it on the login node.\n" + "Default for this machine: {}".format( + default_machine.job_launcher.get_walltime() + ), + ) + + parser.add_argument( + "--job-launcher-extra-args", + help="Extra arguments for the command that launches the\n" + "create_test command.\n" + "(To allow the argument parsing to accept this, enclose the string\n" + 'in quotes, with a leading space, as in " --my-arg foo".)\n' + "Default for this machine: {}".format( + default_machine.job_launcher.get_extra_args() + ), + ) + + parser.add_argument( + "--skip-testroot-creation", + action="store_true", + help="Do not create the directory that will hold the tests.\n" + "This should be used if the desired testroot directory already exists.", + ) + + parser.add_argument( + "--skip-git-status", + action="store_true", + help="Skip printing git and manage_externals status,\n" + "both to screen and to the SRCROOT_GIT_STATUS file in TESTROOT.\n" + "This printing can often be helpful, but this option can be used to\n" + "avoid extraneous output, to reduce the time needed to run this script,\n" + "or if git or manage_externals are currently broken in your sandbox.\n", + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Print what would happen, but do not run any commands.\n" + "(Generally should be run with --verbose.)\n", + ) + + parser.add_argument( + "--machine-name", + default=machine_name, + help="Name of machine for which create_test is run.\n" + "This typically is not needed, but can be provided\n" + "for the sake of testing this script.\n" + "Defaults to current machine: {}".format(machine_name), + ) add_logging_args(parser) @@ -396,32 +503,44 @@ def _commandline_args(): return args + def _check_arg_validity(args): if args.suite_compiler and not args.suite_name: - raise RuntimeError('--suite-compiler can only be specified if using --suite-name') + raise RuntimeError( + "--suite-compiler can only be specified if using --suite-name" + ) if args.rerun_existing_failures and not args.testid_base: - raise RuntimeError('With --rerun-existing-failures, must also specify --testid-base') + raise RuntimeError( + "With --rerun-existing-failures, must also specify --testid-base" + ) + def _get_testid_base(machine_name): """Returns a base testid based on the current date and time and the machine name""" now = datetime.now() now_str = now.strftime("%m%d-%H%M%S") machine_start = machine_name[0:2] - return '{}{}'.format(now_str, machine_start) + return "{}{}".format(now_str, machine_start) + def _get_testroot_base(machine): scratch_dir = machine.scratch_dir if scratch_dir is None: - raise RuntimeError('For a machine without a default specified for the scratch directory, ' - 'must specify --testroot-base') + raise RuntimeError( + "For a machine without a default specified for the scratch directory, " + "must specify --testroot-base" + ) return scratch_dir + def _get_testroot(testroot_base, testid_base): """Get the path to the test root, given a base test id""" return os.path.join(testroot_base, _get_testdir_name(testid_base)) + def _get_testdir_name(testid_base): - return 'tests_{}'.format(testid_base) + return "tests_{}".format(testid_base) + def _make_testroot(testroot, testid_base, dry_run): """Make the testroot directory at the given location, as well as a link in the current @@ -434,38 +553,41 @@ def _make_testroot(testroot, testid_base, dry_run): os.makedirs(testroot) make_link(testroot, _get_testdir_name(testid_base)) + def _record_git_status(testroot, retry, dry_run): """Record git status and related information to stdout and a file""" - output = '' + output = "" ctsm_root = path_to_ctsm_root() output += "create_test --retry: {}\n\n".format(retry) - current_hash = subprocess.check_output(['git', 'show', '--no-patch', - '--format=format:%h (%an, %ad) %s\n', 'HEAD'], - cwd=ctsm_root, - universal_newlines=True) + current_hash = subprocess.check_output( + ["git", "show", "--no-patch", "--format=format:%h (%an, %ad) %s\n", "HEAD"], + cwd=ctsm_root, + universal_newlines=True, + ) output += "Current hash: {}".format(current_hash) - git_status = subprocess.check_output(['git', '-c', 'color.ui=always', - 'status', '--short', '--branch'], - cwd=ctsm_root, - universal_newlines=True) + git_status = subprocess.check_output( + ["git", "-c", "color.ui=always", "status", "--short", "--branch"], + cwd=ctsm_root, + universal_newlines=True, + ) output += git_status - if git_status.count('\n') == 1: + if git_status.count("\n") == 1: # Only line in git status is the branch info output += "(clean sandbox)\n" - manic = os.path.join('manage_externals', 'checkout_externals') - manage_externals_status = subprocess.check_output([manic, '--status', '--verbose'], - cwd=ctsm_root, - universal_newlines=True) - output += 72*'-' + '\n' + 'manage_externals status:' + '\n' + manic = os.path.join("manage_externals", "checkout_externals") + manage_externals_status = subprocess.check_output( + [manic, "--status", "--verbose"], cwd=ctsm_root, universal_newlines=True + ) + output += 72 * "-" + "\n" + "manage_externals status:" + "\n" output += manage_externals_status - output += 72*'-' + '\n' + output += 72 * "-" + "\n" print(output) if not dry_run: - git_status_filepath = os.path.join(testroot, 'SRCROOT_GIT_STATUS') + git_status_filepath = os.path.join(testroot, "SRCROOT_GIT_STATUS") if os.path.exists(git_status_filepath): # If we're reusing an existing directory, it could happen that # SRCROOT_GIT_STATUS already exists. It's still helpful to record the current @@ -473,117 +595,163 @@ def _record_git_status(testroot, retry, dry_run): # make a new file with a date/time-stamp. now = datetime.now() now_str = now.strftime("%m%d-%H%M%S") - git_status_filepath = git_status_filepath + '_' + now_str - with open(git_status_filepath, 'w') as git_status_file: - git_status_file.write(' '.join(sys.argv) + '\n\n') + git_status_filepath = git_status_filepath + "_" + now_str + with open(git_status_filepath, "w") as git_status_file: + git_status_file.write(" ".join(sys.argv) + "\n\n") git_status_file.write("SRCROOT: {}\n".format(ctsm_root)) git_status_file.write(output) -def _get_create_test_args(compare_name, generate_name, baseline_root, - account, walltime, queue, retry, - rerun_existing_failures, - extra_create_test_args): + +def _get_create_test_args( + compare_name, + generate_name, + baseline_root, + account, + walltime, + queue, + retry, + rerun_existing_failures, + extra_create_test_args, +): args = [] if compare_name: - args.extend(['--compare', compare_name]) + args.extend(["--compare", compare_name]) if generate_name: - args.extend(['--generate', generate_name]) + args.extend(["--generate", generate_name]) if baseline_root: - args.extend(['--baseline-root', baseline_root]) + args.extend(["--baseline-root", baseline_root]) if account: - args.extend(['--project', account]) + args.extend(["--project", account]) if walltime: - args.extend(['--walltime', walltime]) + args.extend(["--walltime", walltime]) if queue: - args.extend(['--queue', queue]) - args.extend(['--retry', str(retry)]) + args.extend(["--queue", queue]) + args.extend(["--retry", str(retry)]) if rerun_existing_failures: # In addition to --use-existing, we also need --allow-baseline-overwrite in this # case; otherwise, create_test throws an error saying that the baseline # directories already exist. - args.extend(['--use-existing', '--allow-baseline-overwrite']) + args.extend(["--use-existing", "--allow-baseline-overwrite"]) args.extend(extra_create_test_args.split()) return args + def _make_cs_status_for_suite(testroot, testid_base): """Makes a cs.status file that can be run for the entire test suite""" - testid_pattern = testid_base + '_' + _NUM_COMPILER_CHARS*'?' + testid_pattern = testid_base + "_" + _NUM_COMPILER_CHARS * "?" # The basic cs.status just aggregates results from all of the individual create_tests - create_cs_status(test_root=testroot, - test_id=testid_pattern, - extra_args=_cs_status_xfail_arg(), - filename='cs.status') + create_cs_status( + test_root=testroot, + test_id=testid_pattern, + extra_args=_cs_status_xfail_arg(), + filename="cs.status", + ) # cs.status.fails additionally filters the results so that only failures are shown - create_cs_status(test_root=testroot, - test_id=testid_pattern, - extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()), - filename='cs.status.fails') + create_cs_status( + test_root=testroot, + test_id=testid_pattern, + extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + " " + _cs_status_xfail_arg()), + filename="cs.status.fails", + ) + def _make_cs_status_non_suite(testroot, testid_base): """Makes a cs.status file for a single run of create_test - not a whole test suite""" - create_cs_status(test_root=testroot, - test_id=testid_base, - extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()), - filename='cs.status.fails') + create_cs_status( + test_root=testroot, + test_id=testid_base, + extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + " " + _cs_status_xfail_arg()), + filename="cs.status.fails", + ) + def _cs_status_xfail_arg(): """Returns a string giving the argument to cs_status that will point to CTSM's expected fails xml file """ ctsm_root = path_to_ctsm_root() - xfail_path = os.path.join(ctsm_root, 'cime_config', 'testdefs', 'ExpectedTestFails.xml') + xfail_path = os.path.join( + ctsm_root, "cime_config", "testdefs", "ExpectedTestFails.xml" + ) return "--expected-fails-file {}".format(xfail_path) -def _run_test_suite(cime_path, suite_name, suite_compilers, - machine, testid_base, testroot, create_test_args, - dry_run): + +def _run_test_suite( + cime_path, + suite_name, + suite_compilers, + machine, + testid_base, + testroot, + create_test_args, + dry_run, +): if not suite_compilers: suite_compilers = _get_compilers_for_suite(suite_name, machine.name) for compiler in suite_compilers: - test_args = ['--xml-category', suite_name, - '--xml-machine', machine.name, - '--xml-compiler', compiler] - testid = testid_base + '_' + compiler[0:_NUM_COMPILER_CHARS] - _run_create_test(cime_path=cime_path, - test_args=test_args, machine=machine, - testid=testid, testroot=testroot, - create_test_args=create_test_args, - dry_run=dry_run) + test_args = [ + "--xml-category", + suite_name, + "--xml-machine", + machine.name, + "--xml-compiler", + compiler, + ] + testid = testid_base + "_" + compiler[0:_NUM_COMPILER_CHARS] + _run_create_test( + cime_path=cime_path, + test_args=test_args, + machine=machine, + testid=testid, + testroot=testroot, + create_test_args=create_test_args, + dry_run=dry_run, + ) + def _get_compilers_for_suite(suite_name, machine_name): - test_data = get_tests_from_xml( - xml_machine=machine_name, - xml_category=suite_name) + test_data = get_tests_from_xml(xml_machine=machine_name, xml_category=suite_name) if not test_data: - raise RuntimeError('No tests found for suite {} on machine {}'.format( - suite_name, machine_name)) - compilers = sorted({one_test['compiler'] for one_test in test_data}) + raise RuntimeError( + "No tests found for suite {} on machine {}".format(suite_name, machine_name) + ) + compilers = sorted({one_test["compiler"] for one_test in test_data}) logger.info("Running with compilers: %s", compilers) return compilers -def _run_create_test(cime_path, test_args, machine, testid, testroot, create_test_args, dry_run): - create_test_cmd = _build_create_test_cmd(cime_path=cime_path, - test_args=test_args, - testid=testid, - testroot=testroot, - create_test_args=create_test_args) - stdout_path = os.path.join(testroot, - 'STDOUT.{}'.format(testid)) - stderr_path = os.path.join(testroot, - 'STDERR.{}'.format(testid)) - machine.job_launcher.run_command(create_test_cmd, - stdout_path=stdout_path, - stderr_path=stderr_path, - dry_run=dry_run) + +def _run_create_test( + cime_path, test_args, machine, testid, testroot, create_test_args, dry_run +): + create_test_cmd = _build_create_test_cmd( + cime_path=cime_path, + test_args=test_args, + testid=testid, + testroot=testroot, + create_test_args=create_test_args, + ) + stdout_path = os.path.join(testroot, "STDOUT.{}".format(testid)) + stderr_path = os.path.join(testroot, "STDERR.{}".format(testid)) + machine.job_launcher.run_command( + create_test_cmd, + stdout_path=stdout_path, + stderr_path=stderr_path, + dry_run=dry_run, + ) + def _build_create_test_cmd(cime_path, test_args, testid, testroot, create_test_args): """Builds and returns the create_test command This is a list, where each element of the list is one argument """ - command = [os.path.join(cime_path, 'scripts', 'create_test'), - '--test-id', testid, - '--output-root', testroot] + command = [ + os.path.join(cime_path, "scripts", "create_test"), + "--test-id", + testid, + "--output-root", + testroot, + ] command.extend(test_args) command.extend(create_test_args) return command diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index a1b8d21c15..66f481b159 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -156,7 +156,7 @@ def get_parser(): dest="dom_pft", type=int, default=None, - nargs='*', + nargs="*", ) pt_parser.add_argument( "--pctpft", @@ -165,7 +165,7 @@ def get_parser(): dest="pct_pft", type=float, default=None, - nargs='*', + nargs="*", ) # -- region-specific parser options rg_parser.add_argument( @@ -262,7 +262,7 @@ def get_parser(): subparser.add_argument( "--datm-syr", help="Start year for creating DATM forcing at single point/region. [default: %(" - "default)s]", + "default)s]", action="store", dest="datm_syr", required=False, @@ -272,7 +272,7 @@ def get_parser(): subparser.add_argument( "--datm-eyr", help="End year for creating DATM forcing at single point/region. " - "[default: %(default)s]", + "[default: %(default)s]", action="store", dest="datm_eyr", required=False, @@ -325,6 +325,7 @@ def get_parser(): ) return parser + def setup_user_mods(user_mods_dir, cesmroot): """ Sets up the user mods files and directories @@ -338,14 +339,16 @@ def setup_user_mods(user_mods_dir, cesmroot): for line in basefile: user_file.write(line) - nl_datm_base = os.path.join(cesmroot, "components/cdeps/datm/cime_config" - "/user_nl_datm_streams") + nl_datm_base = os.path.join( + cesmroot, "components/cdeps/datm/cime_config" "/user_nl_datm_streams" + ) nl_datm = os.path.join(user_mods_dir, "user_nl_datm_streams") - with open(nl_datm_base, "r") as base_file, open(nl_datm, 'w') as user_file: + with open(nl_datm_base, "r") as base_file, open(nl_datm, "w") as user_file: for line in base_file: user_file.write(line) -def determine_num_pft (crop): + +def determine_num_pft(crop): """ A simple function to determine the number of pfts. @@ -379,7 +382,7 @@ def setup_files(args, defaults, cesmroot): setup_user_mods(args.user_mods_dir, cesmroot) # DATM data - datm_type = 'datm_gswp3' + datm_type = "datm_gswp3" dir_output_datm = "datmdata" dir_input_datm = defaults.get(datm_type, "dir") if args.create_datm: @@ -391,30 +394,37 @@ def setup_files(args, defaults, cesmroot): # if the crop flag is on - we need to use a different land use and surface data file num_pft = determine_num_pft(args.crop_flag) - fsurf_in = defaults.get("surfdat", "surfdat_"+num_pft+"pft") - fluse_in = defaults.get("landuse", "landuse_"+num_pft+"pft") - - file_dict = {'main_dir': defaults.get("main", "clmforcingindir"), - 'fdomain_in': defaults.get("domain", "file"), - 'fsurf_dir': os.path.join(defaults.get("main", "clmforcingindir"), - os.path.join(defaults.get("surfdat", "dir"))), - 'fluse_dir': os.path.join(defaults.get("main", "clmforcingindir"), - os.path.join(defaults.get("landuse", "dir"))), - 'fsurf_in': fsurf_in, - 'fluse_in': fluse_in, - 'datm_tuple': DatmFiles(dir_input_datm, - dir_output_datm, - defaults.get(datm_type, "domain"), - defaults.get(datm_type, 'solardir'), - defaults.get(datm_type, 'precdir'), - defaults.get(datm_type, 'tpqwdir'), - defaults.get(datm_type, 'solartag'), - defaults.get(datm_type, 'prectag'), - defaults.get(datm_type, 'tpqwtag'), - defaults.get(datm_type, 'solarname'), - defaults.get(datm_type, 'precname'), - defaults.get(datm_type, 'tpqwname')) - } + fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") + fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") + + file_dict = { + "main_dir": defaults.get("main", "clmforcingindir"), + "fdomain_in": defaults.get("domain", "file"), + "fsurf_dir": os.path.join( + defaults.get("main", "clmforcingindir"), + os.path.join(defaults.get("surfdat", "dir")), + ), + "fluse_dir": os.path.join( + defaults.get("main", "clmforcingindir"), + os.path.join(defaults.get("landuse", "dir")), + ), + "fsurf_in": fsurf_in, + "fluse_in": fluse_in, + "datm_tuple": DatmFiles( + dir_input_datm, + dir_output_datm, + defaults.get(datm_type, "domain"), + defaults.get(datm_type, "solardir"), + defaults.get(datm_type, "precdir"), + defaults.get(datm_type, "tpqwdir"), + defaults.get(datm_type, "solartag"), + defaults.get(datm_type, "prectag"), + defaults.get(datm_type, "tpqwtag"), + defaults.get(datm_type, "solarname"), + defaults.get(datm_type, "precname"), + defaults.get(datm_type, "tpqwname"), + ), + } return file_dict @@ -424,46 +434,52 @@ def subset_point(args, file_dict: dict): Subsets surface, domain, land use, and/or DATM files at a single point """ - logger.info("----------------------------------------------------------------------------") + logger.info( + "----------------------------------------------------------------------------" + ) logger.info("This script extracts a single point from the global CTSM datasets.") num_pft = int(determine_num_pft(args.crop_flag)) # -- Create SinglePoint Object single_point = SinglePointCase( - plat = args.plat, - plon = args.plon, - site_name = args.site_name, - create_domain = args.create_domain, - create_surfdata = args.create_surfdata, - create_landuse = args.create_landuse, - create_datm = args.create_datm, - create_user_mods = args.create_user_mods, - dom_pft = args.dom_pft, - pct_pft = args.pct_pft, - num_pft = num_pft, - include_nonveg = args.include_nonveg, - uni_snow = args.uni_snow, - cap_saturation = args.cap_saturation, - out_dir = args.out_dir, - overwrite = args.overwrite, + plat=args.plat, + plon=args.plon, + site_name=args.site_name, + create_domain=args.create_domain, + create_surfdata=args.create_surfdata, + create_landuse=args.create_landuse, + create_datm=args.create_datm, + create_user_mods=args.create_user_mods, + dom_pft=args.dom_pft, + pct_pft=args.pct_pft, + num_pft=num_pft, + include_nonveg=args.include_nonveg, + uni_snow=args.uni_snow, + cap_saturation=args.cap_saturation, + out_dir=args.out_dir, + overwrite=args.overwrite, ) logger.debug(single_point) # -- Create CTSM domain file if single_point.create_domain: - single_point.create_domain_at_point(file_dict["main_dir"], file_dict["fdomain_in"]) + single_point.create_domain_at_point( + file_dict["main_dir"], file_dict["fdomain_in"] + ) # -- Create CTSM surface data file if single_point.create_surfdata: - single_point.create_surfdata_at_point(file_dict["fsurf_dir"], file_dict["fsurf_in"], - args.user_mods_dir) + single_point.create_surfdata_at_point( + file_dict["fsurf_dir"], file_dict["fsurf_in"], args.user_mods_dir + ) # -- Create CTSM transient landuse data file if single_point.create_landuse: - single_point.create_landuse_at_point(file_dict["fluse_dir"], file_dict["fluse_in"], - args.user_mods_dir) + single_point.create_landuse_at_point( + file_dict["fluse_dir"], file_dict["fluse_in"], args.user_mods_dir + ) # -- Create single point atmospheric forcing data if single_point.create_datm: @@ -472,12 +488,15 @@ def subset_point(args, file_dict: dict): # subset the DATM data nl_datm = os.path.join(args.user_mods_dir, "user_nl_datm_streams") - single_point.create_datm_at_point(file_dict['datm_tuple'], args.datm_syr, args.datm_eyr, - nl_datm) + single_point.create_datm_at_point( + file_dict["datm_tuple"], args.datm_syr, args.datm_eyr, nl_datm + ) # -- Write shell commands if single_point.create_user_mods: - single_point.write_shell_commands(os.path.join(args.user_mods_dir, "shell_commands")) + single_point.write_shell_commands( + os.path.join(args.user_mods_dir, "shell_commands") + ) logger.info("Successfully ran script for single point.") @@ -487,23 +506,25 @@ def subset_region(args, file_dict: dict): Subsets surface, domain, land use, and/or DATM files for a region """ - logger.info("----------------------------------------------------------------------------") + logger.info( + "----------------------------------------------------------------------------" + ) logger.info("This script extracts a region from the global CTSM datasets.") # -- Create Region Object region = RegionalCase( - lat1 = args.lat1, - lat2 = args.lat2, - lon1 = args.lon1, - lon2 = args.lon2, - reg_name = args.reg_name, - create_domain = args.create_domain, - create_surfdata = args.create_surfdata, - create_landuse = args.create_landuse, - create_datm = args.create_datm, - create_user_mods = args.create_user_mods, - out_dir = args.out_dir, - overwrite = args.overwrite, + lat1=args.lat1, + lat2=args.lat2, + lon1=args.lon1, + lon2=args.lon2, + reg_name=args.reg_name, + create_domain=args.create_domain, + create_surfdata=args.create_surfdata, + create_landuse=args.create_landuse, + create_datm=args.create_datm, + create_user_mods=args.create_user_mods, + out_dir=args.out_dir, + overwrite=args.overwrite, ) logger.debug(region) @@ -514,13 +535,15 @@ def subset_region(args, file_dict: dict): # -- Create CTSM surface data file if region.create_surfdata: - region.create_surfdata_at_reg(file_dict["fsurf_dir"], file_dict["fsurf_in"], - args.user_mods_dir) + region.create_surfdata_at_reg( + file_dict["fsurf_dir"], file_dict["fsurf_in"], args.user_mods_dir + ) # -- Create CTSM transient landuse data file if region.create_landuse: - region.create_landuse_at_reg(file_dict["fluse_dir"], file_dict["fluse_in"], - args.user_mods_dir) + region.create_landuse_at_reg( + file_dict["fluse_dir"], file_dict["fluse_in"], args.user_mods_dir + ) logger.info("Successfully ran script for a regional case.") @@ -540,20 +563,29 @@ def main(): # --------------------------------- # # print help and exit when no option is chosen if args.run_type != "point" and args.run_type != "region": - err_msg = textwrap.dedent('''\ + err_msg = textwrap.dedent( + """\ \n ------------------------------------ \n Must supply a positional argument: 'point' or 'region'. - ''' - ) + """ + ) raise parser.error(err_msg) - if not any([args.create_surfdata, args.create_domain, args.create_landuse, args.create_datm]): - err_msg = textwrap.dedent('''\ + if not any( + [ + args.create_surfdata, + args.create_domain, + args.create_landuse, + args.create_datm, + ] + ): + err_msg = textwrap.dedent( + """\ \n ------------------------------------ \n Must supply one of: \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n - ''' - ) + """ + ) raise parser.error(err_msg) # --------------------------------- # diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 5754269d59..c573fc7670 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -21,6 +21,7 @@ # readable # pylint: disable=invalid-name + class TestSysFsurdatModifier(unittest.TestCase): """System tests for fsurdat_modifier""" @@ -34,15 +35,19 @@ def setUp(self): - modify_fsurdat.cfg - fsurdat_out.nc """ - self._cfg_template_path = os.path.join(path_to_ctsm_root(), - 'tools/modify_fsurdat/modify_template.cfg') - testinputs_path = os.path.join(path_to_ctsm_root(), - 'python/ctsm/test/testinputs') - self._fsurdat_in = os.path.join(testinputs_path, - 'surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc') + self._cfg_template_path = os.path.join( + path_to_ctsm_root(), "tools/modify_fsurdat/modify_template.cfg" + ) + testinputs_path = os.path.join( + path_to_ctsm_root(), "python/ctsm/test/testinputs" + ) + self._fsurdat_in = os.path.join( + testinputs_path, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", + ) self._tempdir = tempfile.mkdtemp() - self._cfg_file_path = os.path.join(self._tempdir, 'modify_fsurdat.cfg') - self._fsurdat_out = os.path.join(self._tempdir, 'fsurdat_out.nc') + self._cfg_file_path = os.path.join(self._tempdir, "modify_fsurdat.cfg") + self._fsurdat_out = os.path.join(self._tempdir, "fsurdat_out.nc") def tearDown(self): """ @@ -68,7 +73,6 @@ def test_minimalInfo(self): # assert that fsurdat_out equals fsurdat_in self.assertTrue(fsurdat_out_data.equals(fsurdat_in_data)) - def test_crop(self): """ This version replaces the vegetation with a crop @@ -89,13 +93,13 @@ def test_crop(self): self.assertFalse(fsurdat_out_data.equals(fsurdat_in_data)) # compare fsurdat_out to fsurdat_out_baseline located in /testinputs - fsurdat_out_baseline = self._fsurdat_in[:-3] + '_modified_with_crop' + \ - self._fsurdat_in[-3:] + fsurdat_out_baseline = ( + self._fsurdat_in[:-3] + "_modified_with_crop" + self._fsurdat_in[-3:] + ) fsurdat_out_base_data = xr.open_dataset(fsurdat_out_baseline) # assert that fsurdat_out equals fsurdat_out_baseline self.assertTrue(fsurdat_out_data.equals(fsurdat_out_base_data)) - def test_allInfo(self): """ This version specifies all possible information @@ -116,105 +120,103 @@ def test_allInfo(self): self.assertFalse(fsurdat_out_data.equals(fsurdat_in_data)) # compare fsurdat_out to fsurdat_out_baseline located in /testinputs - fsurdat_out_baseline = self._fsurdat_in[:-3] + '_modified' + \ - self._fsurdat_in[-3:] + fsurdat_out_baseline = ( + self._fsurdat_in[:-3] + "_modified" + self._fsurdat_in[-3:] + ) fsurdat_out_base_data = xr.open_dataset(fsurdat_out_baseline) # assert that fsurdat_out equals fsurdat_out_baseline self.assertTrue(fsurdat_out_data.equals(fsurdat_out_base_data)) - def _create_config_file_minimal(self): """ Open the new and the template .cfg files Loop line by line through the template .cfg file When string matches, replace that line's content """ - with open (self._cfg_file_path, 'w', encoding='utf-8') as cfg_out: - with open (self._cfg_template_path, 'r', encoding='utf-8') as cfg_in: + with open(self._cfg_file_path, "w", encoding="utf-8") as cfg_out: + with open(self._cfg_template_path, "r", encoding="utf-8") as cfg_in: for line in cfg_in: - if re.match(r' *fsurdat_in *=', line): - line = f'fsurdat_in = {self._fsurdat_in}' - elif re.match(r' *fsurdat_out *=', line): - line = f'fsurdat_out = {self._fsurdat_out}' + if re.match(r" *fsurdat_in *=", line): + line = f"fsurdat_in = {self._fsurdat_in}" + elif re.match(r" *fsurdat_out *=", line): + line = f"fsurdat_out = {self._fsurdat_out}" cfg_out.write(line) - def _create_config_file_crop(self): """ Open the new and the template .cfg files Loop line by line through the template .cfg file When string matches, replace that line's content """ - with open (self._cfg_file_path, 'w', encoding='utf-8') as cfg_out: - with open (self._cfg_template_path, 'r', encoding='utf-8') as cfg_in: + with open(self._cfg_file_path, "w", encoding="utf-8") as cfg_out: + with open(self._cfg_template_path, "r", encoding="utf-8") as cfg_in: for line in cfg_in: - if re.match(r' *fsurdat_in *=', line): - line = f'fsurdat_in = {self._fsurdat_in}' - elif re.match(r' *fsurdat_out *=', line): - line = f'fsurdat_out = {self._fsurdat_out}' - elif re.match(r' *lnd_lat_1 *=', line): - line = 'lnd_lat_1 = -10\n' - elif re.match(r' *lnd_lat_2 *=', line): - line = 'lnd_lat_2 = -7\n' - elif re.match(r' *lnd_lon_1 *=', line): - line = 'lnd_lon_1 = 295\n' - elif re.match(r' *lnd_lon_2 *=', line): - line = 'lnd_lon_2 = 300\n' - elif re.match(r' *dom_plant *=', line): - line = 'dom_plant = 15' - elif re.match(r' *lai *=', line): - line = 'lai = 0 1 2 3 4 5 5 4 3 2 1 0\n' - elif re.match(r' *sai *=', line): - line = 'sai = 1 1 1 1 1 1 1 1 1 1 1 1\n' - elif re.match(r' *hgt_top *=', line): - line = 'hgt_top = 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5\n' - elif re.match(r' *hgt_bot *=', line): - line = 'hgt_bot = 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1\n' + if re.match(r" *fsurdat_in *=", line): + line = f"fsurdat_in = {self._fsurdat_in}" + elif re.match(r" *fsurdat_out *=", line): + line = f"fsurdat_out = {self._fsurdat_out}" + elif re.match(r" *lnd_lat_1 *=", line): + line = "lnd_lat_1 = -10\n" + elif re.match(r" *lnd_lat_2 *=", line): + line = "lnd_lat_2 = -7\n" + elif re.match(r" *lnd_lon_1 *=", line): + line = "lnd_lon_1 = 295\n" + elif re.match(r" *lnd_lon_2 *=", line): + line = "lnd_lon_2 = 300\n" + elif re.match(r" *dom_plant *=", line): + line = "dom_plant = 15" + elif re.match(r" *lai *=", line): + line = "lai = 0 1 2 3 4 5 5 4 3 2 1 0\n" + elif re.match(r" *sai *=", line): + line = "sai = 1 1 1 1 1 1 1 1 1 1 1 1\n" + elif re.match(r" *hgt_top *=", line): + line = "hgt_top = 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5\n" + elif re.match(r" *hgt_bot *=", line): + line = "hgt_bot = 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1\n" cfg_out.write(line) - def _create_config_file_complete(self): """ Open the new and the template .cfg files Loop line by line through the template .cfg file When string matches, replace that line's content """ - with open (self._cfg_file_path, 'w', encoding='utf-8') as cfg_out: - with open (self._cfg_template_path, 'r', encoding='utf-8') as cfg_in: + with open(self._cfg_file_path, "w", encoding="utf-8") as cfg_out: + with open(self._cfg_template_path, "r", encoding="utf-8") as cfg_in: for line in cfg_in: - if re.match(r' *fsurdat_in *=', line): - line = f'fsurdat_in = {self._fsurdat_in}' - elif re.match(r' *fsurdat_out *=', line): - line = f'fsurdat_out = {self._fsurdat_out}' - elif re.match(r' *idealized *=', line): - line = 'idealized = True' - elif re.match(r' *lnd_lat_1 *=', line): - line = 'lnd_lat_1 = -10\n' - elif re.match(r' *lnd_lat_2 *=', line): - line = 'lnd_lat_2 = -7\n' - elif re.match(r' *lnd_lon_1 *=', line): - line = 'lnd_lon_1 = 295\n' - elif re.match(r' *lnd_lon_2 *=', line): - line = 'lnd_lon_2 = 300\n' - elif re.match(r' *dom_plant *=', line): - line = 'dom_plant = 1' - elif re.match(r' *lai *=', line): - line = 'lai = 0 1 2 3 4 5 5 4 3 2 1 0\n' - elif re.match(r' *sai *=', line): - line = 'sai = 1 1 1 1 1 1 1 1 1 1 1 1\n' - elif re.match(r' *hgt_top *=', line): - line = 'hgt_top = 5 5 5 5 5 5 5 5 5 5 5 5\n' - elif re.match(r' *hgt_bot *=', line): - line = 'hgt_bot = 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5\n' - elif re.match(r' *soil_color *=', line): - line = 'soil_color = 5' - elif re.match(r' *std_elev *=', line): - line = 'std_elev = 0.1' - elif re.match(r' *max_sat_area *=', line): - line = 'max_sat_area = 0.2' + if re.match(r" *fsurdat_in *=", line): + line = f"fsurdat_in = {self._fsurdat_in}" + elif re.match(r" *fsurdat_out *=", line): + line = f"fsurdat_out = {self._fsurdat_out}" + elif re.match(r" *idealized *=", line): + line = "idealized = True" + elif re.match(r" *lnd_lat_1 *=", line): + line = "lnd_lat_1 = -10\n" + elif re.match(r" *lnd_lat_2 *=", line): + line = "lnd_lat_2 = -7\n" + elif re.match(r" *lnd_lon_1 *=", line): + line = "lnd_lon_1 = 295\n" + elif re.match(r" *lnd_lon_2 *=", line): + line = "lnd_lon_2 = 300\n" + elif re.match(r" *dom_plant *=", line): + line = "dom_plant = 1" + elif re.match(r" *lai *=", line): + line = "lai = 0 1 2 3 4 5 5 4 3 2 1 0\n" + elif re.match(r" *sai *=", line): + line = "sai = 1 1 1 1 1 1 1 1 1 1 1 1\n" + elif re.match(r" *hgt_top *=", line): + line = "hgt_top = 5 5 5 5 5 5 5 5 5 5 5 5\n" + elif re.match(r" *hgt_bot *=", line): + line = "hgt_bot = 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5\n" + elif re.match(r" *soil_color *=", line): + line = "soil_color = 5" + elif re.match(r" *std_elev *=", line): + line = "std_elev = 0.1" + elif re.match(r" *max_sat_area *=", line): + line = "max_sat_area = 0.2" cfg_out.write(line) -if __name__ == '__main__': +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_args_utils.py b/python/ctsm/test/test_unit_args_utils.py index 73aab9f6a8..3e7ec122bc 100755 --- a/python/ctsm/test/test_unit_args_utils.py +++ b/python/ctsm/test/test_unit_args_utils.py @@ -13,15 +13,17 @@ # -- add python/ctsm to path (needed if we want to run the test stand-alone) _CTSM_PYTHON = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir +) sys.path.insert(1, _CTSM_PYTHON) -#pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position from ctsm.args_utils import plon_type, plat_type from ctsm import unit_testing # pylint: disable=invalid-name + class TestArgsPlon(unittest.TestCase): """ Tests for plot_type in args_util.py @@ -87,10 +89,12 @@ def test_plonType_positive_360(self): result = plon_type(360) self.assertEqual(result, 360.0) + class TestArgsPlat(unittest.TestCase): """ Tests for plat_type in args_util.py """ + def test_platType_outOfBounds_positive(self): """ Test of plat_type bigger than 90 @@ -123,6 +127,7 @@ def test_platType_outOfBounds_negative(self): ): _ = plat_type(-91) + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_machine.py b/python/ctsm/test/test_unit_machine.py index bc1a6f777d..d7aa6aca3d 100755 --- a/python/ctsm/test/test_unit_machine.py +++ b/python/ctsm/test/test_unit_machine.py @@ -6,27 +6,41 @@ import unittest import os -from ctsm import add_cime_to_path # pylint: disable=unused-import +from ctsm import add_cime_to_path # pylint: disable=unused-import from ctsm import unit_testing -from ctsm.machine import (create_machine, get_possibly_overridden_mach_value, - CREATE_TEST_QUEUE_UNSPECIFIED) +from ctsm.machine import ( + create_machine, + get_possibly_overridden_mach_value, + CREATE_TEST_QUEUE_UNSPECIFIED, +) from ctsm.machine_utils import get_user from ctsm.machine_defaults import MACHINE_DEFAULTS, MachineDefaults, QsubDefaults from ctsm.joblauncher.job_launcher_no_batch import JobLauncherNoBatch from ctsm.joblauncher.job_launcher_qsub import JobLauncherQsub -from ctsm.joblauncher.job_launcher_factory import JOB_LAUNCHER_QSUB, JOB_LAUNCHER_NOBATCH +from ctsm.joblauncher.job_launcher_factory import ( + JOB_LAUNCHER_QSUB, + JOB_LAUNCHER_NOBATCH, +) # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names # pylint: disable=invalid-name + class TestCreateMachine(unittest.TestCase): """Tests of create_machine""" - def assertMachineInfo(self, machine, name, scratch_dir, baseline_dir, account, - create_test_retry=0, - create_test_queue=CREATE_TEST_QUEUE_UNSPECIFIED): + def assertMachineInfo( + self, + machine, + name, + scratch_dir, + baseline_dir, + account, + create_test_retry=0, + create_test_queue=CREATE_TEST_QUEUE_UNSPECIFIED, + ): """Asserts that the basic machine info is as expected. This does NOT dive down into the job launcher""" @@ -46,7 +60,9 @@ def assertNoBatchInfo(self, machine, nice_level=None): nice_level = 0 self.assertEqual(launcher.get_nice_level(), nice_level) - def assertQsubInfo(self, machine, queue, walltime, account, required_args, extra_args): + def assertQsubInfo( + self, machine, queue, walltime, account, required_args, extra_args + ): """Asserts that the machine's launcher is of type JobLauncherQsub, and has values as expected""" launcher = machine.job_launcher @@ -62,114 +78,138 @@ def create_defaults(default_job_launcher=JOB_LAUNCHER_QSUB): """Creates test-specific defaults so we don't tie the tests to changes in the real defaults""" defaults = { - 'cheyenne': MachineDefaults( + "cheyenne": MachineDefaults( job_launcher_type=default_job_launcher, - scratch_dir=os.path.join(os.path.sep, 'glade', 'scratch', get_user()), - baseline_dir=os.path.join(os.path.sep, 'my', 'baselines'), + scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), + baseline_dir=os.path.join(os.path.sep, "my", "baselines"), account_required=True, create_test_retry=2, create_test_queue="regular", job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue='regular', - walltime='06:00:00', - extra_args='', - required_args= - '-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login') - }) - } + queue="regular", + walltime="06:00:00", + extra_args="", + required_args="-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login", + ) + }, + ) + } return defaults def test_unknownMachine_defaults(self): """Tests a machine not in the defaults structure, with no overriding arguments""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - account='a123') - self.assertMachineInfo(machine=machine, - name='unknown_test_machine', - scratch_dir=None, - baseline_dir=None, - account='a123') + machine = create_machine( + "unknown_test_machine", MACHINE_DEFAULTS, account="a123" + ) + self.assertMachineInfo( + machine=machine, + name="unknown_test_machine", + scratch_dir=None, + baseline_dir=None, + account="a123", + ) self.assertNoBatchInfo(machine) def test_noBatchMachine_niceLevel(self): """Tests a no-batch machine where the nice level is explicit""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - job_launcher_type=JOB_LAUNCHER_NOBATCH, - scratch_dir='/path/to/scratch', - account='a123', - job_launcher_nice_level=13) - self.assertMachineInfo(machine=machine, - name='unknown_test_machine', - scratch_dir='/path/to/scratch', - baseline_dir=None, - account='a123') + machine = create_machine( + "unknown_test_machine", + MACHINE_DEFAULTS, + job_launcher_type=JOB_LAUNCHER_NOBATCH, + scratch_dir="/path/to/scratch", + account="a123", + job_launcher_nice_level=13, + ) + self.assertMachineInfo( + machine=machine, + name="unknown_test_machine", + scratch_dir="/path/to/scratch", + baseline_dir=None, + account="a123", + ) self.assertNoBatchInfo(machine, nice_level=13) def test_unknownMachine_argsExplicit(self): """Tests a machine not in the defaults structure, with explicit arguments""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - scratch_dir='/path/to/scratch', - job_launcher_type=JOB_LAUNCHER_QSUB, - account='a123', - job_launcher_queue='my_queue', - job_launcher_walltime='1:23:45', - job_launcher_extra_args='--some args') - self.assertMachineInfo(machine=machine, - name='unknown_test_machine', - scratch_dir='/path/to/scratch', - baseline_dir=None, - account='a123') - self.assertQsubInfo(machine=machine, - queue='my_queue', - walltime='1:23:45', - account='a123', - required_args='', - extra_args='--some args') + machine = create_machine( + "unknown_test_machine", + MACHINE_DEFAULTS, + scratch_dir="/path/to/scratch", + job_launcher_type=JOB_LAUNCHER_QSUB, + account="a123", + job_launcher_queue="my_queue", + job_launcher_walltime="1:23:45", + job_launcher_extra_args="--some args", + ) + self.assertMachineInfo( + machine=machine, + name="unknown_test_machine", + scratch_dir="/path/to/scratch", + baseline_dir=None, + account="a123", + ) + self.assertQsubInfo( + machine=machine, + queue="my_queue", + walltime="1:23:45", + account="a123", + required_args="", + extra_args="--some args", + ) def test_knownMachine_defaults(self): """Tests a machine known in the defaults structure, with no overriding arguments""" defaults = self.create_defaults() - machine = create_machine('cheyenne', defaults, account='a123') - self.assertMachineInfo(machine=machine, - name='cheyenne', - scratch_dir=os.path.join(os.path.sep, - 'glade', - 'scratch', - get_user()), - baseline_dir=os.path.join(os.path.sep, 'my', 'baselines'), - account='a123', - create_test_retry=2, - create_test_queue="regular") - self.assertQsubInfo(machine=machine, - queue='regular', - walltime='06:00:00', - account='a123', - required_args='-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login', - extra_args='') + machine = create_machine("cheyenne", defaults, account="a123") + self.assertMachineInfo( + machine=machine, + name="cheyenne", + scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), + baseline_dir=os.path.join(os.path.sep, "my", "baselines"), + account="a123", + create_test_retry=2, + create_test_queue="regular", + ) + self.assertQsubInfo( + machine=machine, + queue="regular", + walltime="06:00:00", + account="a123", + required_args="-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login", + extra_args="", + ) def test_knownMachine_argsExplicit(self): """Tests a machine known in the defaults structure, with explicit arguments""" defaults = self.create_defaults(default_job_launcher=JOB_LAUNCHER_NOBATCH) - machine = create_machine('cheyenne', defaults, - job_launcher_type=JOB_LAUNCHER_QSUB, - scratch_dir='/custom/path/to/scratch', - account='a123', - job_launcher_queue='custom_queue', - job_launcher_walltime='9:87:65', - job_launcher_extra_args='--custom args') - self.assertMachineInfo(machine=machine, - name='cheyenne', - scratch_dir='/custom/path/to/scratch', - baseline_dir=os.path.join(os.path.sep, 'my', 'baselines'), - account='a123', - create_test_retry=2, - create_test_queue="regular") - self.assertQsubInfo(machine=machine, - queue='custom_queue', - walltime='9:87:65', - account='a123', - required_args='-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login', - extra_args='--custom args') + machine = create_machine( + "cheyenne", + defaults, + job_launcher_type=JOB_LAUNCHER_QSUB, + scratch_dir="/custom/path/to/scratch", + account="a123", + job_launcher_queue="custom_queue", + job_launcher_walltime="9:87:65", + job_launcher_extra_args="--custom args", + ) + self.assertMachineInfo( + machine=machine, + name="cheyenne", + scratch_dir="/custom/path/to/scratch", + baseline_dir=os.path.join(os.path.sep, "my", "baselines"), + account="a123", + create_test_retry=2, + create_test_queue="regular", + ) + self.assertQsubInfo( + machine=machine, + queue="custom_queue", + walltime="9:87:65", + account="a123", + required_args="-l select=1:ncpus=36:mpiprocs=1 -r n -l inception=login", + extra_args="--custom args", + ) # ------------------------------------------------------------------------ # Tests of get_possibly_overridden_mach_value @@ -178,31 +218,33 @@ def test_knownMachine_argsExplicit(self): def test_baselineDir_overridden(self): """Tests get_possibly_overridden_mach_value when baseline_dir is provided""" defaults = self.create_defaults() - machine = create_machine('cheyenne', defaults, account='a123') - baseline_dir = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value='mypath') - self.assertEqual(baseline_dir, 'mypath') + machine = create_machine("cheyenne", defaults, account="a123") + baseline_dir = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value="mypath" + ) + self.assertEqual(baseline_dir, "mypath") def test_baselineDir_default(self): """Tests get_possibly_overridden_mach_value when baseline_dir is not provided""" defaults = self.create_defaults() - machine = create_machine('cheyenne', defaults, account='a123') - baseline_dir = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value=None) - self.assertEqual(baseline_dir, os.path.join(os.path.sep, 'my', 'baselines')) + machine = create_machine("cheyenne", defaults, account="a123") + baseline_dir = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value=None + ) + self.assertEqual(baseline_dir, os.path.join(os.path.sep, "my", "baselines")) def test_baselineDir_noDefault(self): """Tests get_possibly_overridden_mach_value when baseline_dir is not provided and there is no default""" - machine = create_machine('unknown_test_machine', MACHINE_DEFAULTS, - account='a123') - baseline_dir = get_possibly_overridden_mach_value(machine, - varname='baseline_dir', - value=None) + machine = create_machine( + "unknown_test_machine", MACHINE_DEFAULTS, account="a123" + ) + baseline_dir = get_possibly_overridden_mach_value( + machine, varname="baseline_dir", value=None + ) self.assertIsNone(baseline_dir) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index a5182ae6e4..dca8dfa99f 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -22,7 +22,7 @@ class TestModifyFsurdat(unittest.TestCase): """Tests the setvar_lev functions and the - _get_rectangle function + _get_rectangle function """ def test_setvarLev(self): @@ -122,10 +122,15 @@ def test_getNotRectangle_lon1leLon2Lat1leLat2(self): lat_1 = 6 lat_2 = 8 # lat_1 < lat_2 rectangle = ModifyFsurdat._get_rectangle( - lon_1=lon_1, lon_2=lon_2, lat_1=lat_1, lat_2=lat_2, - longxy=longxy, latixy=latixy) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=longxy, + latixy=latixy, + ) not_rectangle = np.logical_not(rectangle) - compare = np.ones((rows,cols)) + compare = np.ones((rows, cols)) # assert this to confirm intuitive understanding of these matrices self.assertEqual(np.size(not_rectangle), np.size(compare)) @@ -163,10 +168,15 @@ def test_getNotRectangle_lon1leLon2Lat1gtLat2(self): lat_1 = 4 lat_2 = 0 # lat_1 > lat_2 rectangle = ModifyFsurdat._get_rectangle( - lon_1=lon_1, lon_2=lon_2, lat_1=lat_1, lat_2=lat_2, - longxy=longxy, latixy=latixy) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=longxy, + latixy=latixy, + ) not_rectangle = np.logical_not(rectangle) - compare = np.ones((rows,cols)) + compare = np.ones((rows, cols)) # assert this to confirm intuitive understanding of these matrices self.assertEqual(np.size(not_rectangle), np.size(compare)) @@ -203,10 +213,15 @@ def test_getNotRectangle_lon1gtLon2Lat1leLat2(self): lat_1 = 2 lat_2 = 3 # lat_1 < lat_2 rectangle = ModifyFsurdat._get_rectangle( - lon_1=lon_1, lon_2=lon_2, lat_1=lat_1, lat_2=lat_2, - longxy=longxy, latixy=latixy) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=longxy, + latixy=latixy, + ) not_rectangle = np.logical_not(rectangle) - compare = np.ones((rows,cols)) + compare = np.ones((rows, cols)) # assert this to confirm intuitive understanding of these matrices self.assertEqual(np.size(not_rectangle), np.size(compare)) @@ -243,10 +258,15 @@ def test_getNotRectangle_lon1gtLon2Lat1gtLat2(self): lat_1 = 0 lat_2 = -3 # lat_1 > lat_2 rectangle = ModifyFsurdat._get_rectangle( - lon_1=lon_1, lon_2=lon_2, lat_1=lat_1, lat_2=lat_2, - longxy=longxy, latixy=latixy) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=longxy, + latixy=latixy, + ) not_rectangle = np.logical_not(rectangle) - compare = np.ones((rows,cols)) + compare = np.ones((rows, cols)) # assert this to confirm intuitive understanding of these matrices self.assertEqual(np.size(not_rectangle), np.size(compare)) @@ -285,10 +305,15 @@ def test_getNotRectangle_lonsStraddle0deg(self): lat_1 = -4 lat_2 = -6 # lat_1 > lat_2 rectangle = ModifyFsurdat._get_rectangle( - lon_1=lon_1, lon_2=lon_2, lat_1=lat_1, lat_2=lat_2, - longxy=longxy, latixy=latixy) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=longxy, + latixy=latixy, + ) not_rectangle = np.logical_not(rectangle) - compare = np.ones((rows,cols)) + compare = np.ones((rows, cols)) # assert this to confirm intuitive understanding of these matrices self.assertEqual(np.size(not_rectangle), np.size(compare)) @@ -322,11 +347,17 @@ def test_getNotRectangle_latsOutOfBounds(self): lon_2 = 5 lat_1 = -91 lat_2 = 91 - with self.assertRaisesRegex(SystemExit, - "lat_1 and lat_2 need to be in the range -90 to 90"): + with self.assertRaisesRegex( + SystemExit, "lat_1 and lat_2 need to be in the range -90 to 90" + ): _ = ModifyFsurdat._get_rectangle( - lon_1=lon_1, lon_2=lon_2, lat_1=lat_1, lat_2=lat_2, - longxy=longxy, latixy=latixy) + lon_1=lon_1, + lon_2=lon_2, + lat_1=lat_1, + lat_2=lat_2, + longxy=longxy, + latixy=latixy, + ) def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ diff --git a/python/ctsm/test/test_unit_run_sys_tests.py b/python/ctsm/test/test_unit_run_sys_tests.py index 218001c7f7..ba5b1b0ed1 100755 --- a/python/ctsm/test/test_unit_run_sys_tests.py +++ b/python/ctsm/test/test_unit_run_sys_tests.py @@ -14,7 +14,7 @@ import six from six_additions import mock, assertNotRegex -from ctsm import add_cime_to_path # pylint: disable=unused-import +from ctsm import add_cime_to_path # pylint: disable=unused-import from ctsm import unit_testing from ctsm.run_sys_tests import run_sys_tests from ctsm.machine_defaults import MACHINE_DEFAULTS @@ -26,17 +26,17 @@ # pylint: disable=invalid-name # Replace the slow _record_git_status with a fake that does nothing -@mock.patch('ctsm.run_sys_tests._record_git_status', mock.MagicMock(return_value=None)) +@mock.patch("ctsm.run_sys_tests._record_git_status", mock.MagicMock(return_value=None)) class TestRunSysTests(unittest.TestCase): """Tests of run_sys_tests""" - _MACHINE_NAME = 'fake_machine' + _MACHINE_NAME = "fake_machine" def setUp(self): self._original_wd = os.getcwd() self._curdir = tempfile.mkdtemp() os.chdir(self._curdir) - self._scratch = os.path.join(self._curdir, 'scratch') + self._scratch = os.path.join(self._curdir, "scratch") os.makedirs(self._scratch) def tearDown(self): @@ -44,51 +44,47 @@ def tearDown(self): shutil.rmtree(self._curdir, ignore_errors=True) def _make_machine(self, account=None): - machine = create_machine(machine_name=self._MACHINE_NAME, - defaults=MACHINE_DEFAULTS, - job_launcher_type=JOB_LAUNCHER_FAKE, - scratch_dir=self._scratch, - account=account) + machine = create_machine( + machine_name=self._MACHINE_NAME, + defaults=MACHINE_DEFAULTS, + job_launcher_type=JOB_LAUNCHER_FAKE, + scratch_dir=self._scratch, + account=account, + ) return machine @staticmethod def _fake_now(): - return datetime(year=2001, - month=2, - day=3, - hour=4, - minute=5, - second=6) + return datetime(year=2001, month=2, day=3, hour=4, minute=5, second=6) def _cime_path(self): # For the sake of paths to scripts used in run_sys_tests: Pretend that cime exists # under the current directory, even though it doesn't - return os.path.join(self._curdir, 'cime') + return os.path.join(self._curdir, "cime") @staticmethod def _expected_testid(): """Returns an expected testid based on values set in _fake_now and _MACHINE_NAME""" - return '0203-040506fa' + return "0203-040506fa" def _expected_testroot(self): """Returns an expected testroot based on values set in _fake_now and _MACHINE_NAME This just returns the name of the testroot directory, not the full path""" - return 'tests_{}'.format(self._expected_testid()) + return "tests_{}".format(self._expected_testid()) def test_testroot_setup(self): """Ensure that the appropriate test root directory is created and populated""" machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.datetime') as mock_date: + with mock.patch("ctsm.run_sys_tests.datetime") as mock_date: mock_date.now.side_effect = self._fake_now - run_sys_tests(machine=machine, cime_path=self._cime_path(), - testlist=['foo']) + run_sys_tests( + machine=machine, cime_path=self._cime_path(), testlist=["foo"] + ) - expected_dir = os.path.join(self._scratch, - self._expected_testroot()) + expected_dir = os.path.join(self._scratch, self._expected_testroot()) self.assertTrue(os.path.isdir(expected_dir)) - expected_link = os.path.join(self._curdir, - self._expected_testroot()) + expected_link = os.path.join(self._curdir, self._expected_testroot()) self.assertTrue(os.path.islink(expected_link)) self.assertEqual(os.readlink(expected_link), expected_dir) @@ -106,31 +102,40 @@ def test_createTestCommand_testnames(self): (3) That a cs.status.fails file was created """ machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.datetime') as mock_date: + with mock.patch("ctsm.run_sys_tests.datetime") as mock_date: mock_date.now.side_effect = self._fake_now - run_sys_tests(machine=machine, cime_path=self._cime_path(), - testlist=['test1', 'test2']) + run_sys_tests( + machine=machine, + cime_path=self._cime_path(), + testlist=["test1", "test2"], + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 1) command = all_commands[0].cmd - expected_create_test = os.path.join(self._cime_path(), 'scripts', 'create_test') - six.assertRegex(self, command, r'^ *{}\s'.format(re.escape(expected_create_test))) - six.assertRegex(self, command, r'--test-id +{}\s'.format(self._expected_testid())) + expected_create_test = os.path.join(self._cime_path(), "scripts", "create_test") + six.assertRegex( + self, command, r"^ *{}\s".format(re.escape(expected_create_test)) + ) + six.assertRegex( + self, command, r"--test-id +{}\s".format(self._expected_testid()) + ) expected_testroot_path = os.path.join(self._scratch, self._expected_testroot()) - six.assertRegex(self, command, r'--output-root +{}\s'.format(expected_testroot_path)) - six.assertRegex(self, command, r'--retry +0(\s|$)') - six.assertRegex(self, command, r'test1 +test2(\s|$)') - assertNotRegex(self, command, r'--compare\s') - assertNotRegex(self, command, r'--generate\s') - assertNotRegex(self, command, r'--baseline-root\s') + six.assertRegex( + self, command, r"--output-root +{}\s".format(expected_testroot_path) + ) + six.assertRegex(self, command, r"--retry +0(\s|$)") + six.assertRegex(self, command, r"test1 +test2(\s|$)") + assertNotRegex(self, command, r"--compare\s") + assertNotRegex(self, command, r"--generate\s") + assertNotRegex(self, command, r"--baseline-root\s") # In the machine object for this test, create_test_queue will be 'unspecified'; # verify that this results in there being no '--queue' argument: - assertNotRegex(self, command, r'--queue\s') + assertNotRegex(self, command, r"--queue\s") - expected_cs_status = os.path.join(self._scratch, - self._expected_testroot(), - 'cs.status.fails') + expected_cs_status = os.path.join( + self._scratch, self._expected_testroot(), "cs.status.fails" + ) self.assertTrue(os.path.isfile(expected_cs_status)) def test_createTestCommand_testfileAndExtraArgs(self): @@ -144,38 +149,42 @@ def test_createTestCommand_testfileAndExtraArgs(self): (3) That a cs.status.fails file was created """ - machine = self._make_machine(account='myaccount') - testroot_base = os.path.join(self._scratch, 'my', 'testroot') - run_sys_tests(machine=machine, cime_path=self._cime_path(), - testfile='/path/to/testfile', - testid_base='mytestid', - testroot_base=testroot_base, - compare_name='mycompare', - generate_name='mygenerate', - baseline_root='myblroot', - walltime='3:45:67', - queue='runqueue', - retry=5, - extra_create_test_args='--some extra --createtest args') + machine = self._make_machine(account="myaccount") + testroot_base = os.path.join(self._scratch, "my", "testroot") + run_sys_tests( + machine=machine, + cime_path=self._cime_path(), + testfile="/path/to/testfile", + testid_base="mytestid", + testroot_base=testroot_base, + compare_name="mycompare", + generate_name="mygenerate", + baseline_root="myblroot", + walltime="3:45:67", + queue="runqueue", + retry=5, + extra_create_test_args="--some extra --createtest args", + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 1) command = all_commands[0].cmd - six.assertRegex(self, command, r'--test-id +mytestid(\s|$)') - expected_testroot = os.path.join(testroot_base, 'tests_mytestid') - six.assertRegex(self, command, r'--output-root +{}(\s|$)'.format(expected_testroot)) - six.assertRegex(self, command, r'--testfile +/path/to/testfile(\s|$)') - six.assertRegex(self, command, r'--compare +mycompare(\s|$)') - six.assertRegex(self, command, r'--generate +mygenerate(\s|$)') - six.assertRegex(self, command, r'--baseline-root +myblroot(\s|$)') - six.assertRegex(self, command, r'--walltime +3:45:67(\s|$)') - six.assertRegex(self, command, r'--queue +runqueue(\s|$)') - six.assertRegex(self, command, r'--project +myaccount(\s|$)') - six.assertRegex(self, command, r'--retry +5(\s|$)') - six.assertRegex(self, command, r'--some +extra +--createtest +args(\s|$)') - - expected_cs_status = os.path.join(expected_testroot, - 'cs.status.fails') + six.assertRegex(self, command, r"--test-id +mytestid(\s|$)") + expected_testroot = os.path.join(testroot_base, "tests_mytestid") + six.assertRegex( + self, command, r"--output-root +{}(\s|$)".format(expected_testroot) + ) + six.assertRegex(self, command, r"--testfile +/path/to/testfile(\s|$)") + six.assertRegex(self, command, r"--compare +mycompare(\s|$)") + six.assertRegex(self, command, r"--generate +mygenerate(\s|$)") + six.assertRegex(self, command, r"--baseline-root +myblroot(\s|$)") + six.assertRegex(self, command, r"--walltime +3:45:67(\s|$)") + six.assertRegex(self, command, r"--queue +runqueue(\s|$)") + six.assertRegex(self, command, r"--project +myaccount(\s|$)") + six.assertRegex(self, command, r"--retry +5(\s|$)") + six.assertRegex(self, command, r"--some +extra +--createtest +args(\s|$)") + + expected_cs_status = os.path.join(expected_testroot, "cs.status.fails") self.assertTrue(os.path.isfile(expected_cs_status)) def test_createTestCommands_testsuite(self): @@ -189,77 +198,100 @@ def test_createTestCommands_testsuite(self): It also ensures that the cs.status.fails and cs.status files are created """ machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.datetime') as mock_date, \ - mock.patch('ctsm.run_sys_tests.get_tests_from_xml') as mock_get_tests: + with mock.patch("ctsm.run_sys_tests.datetime") as mock_date, mock.patch( + "ctsm.run_sys_tests.get_tests_from_xml" + ) as mock_get_tests: mock_date.now.side_effect = self._fake_now - mock_get_tests.return_value = [{'compiler': 'intel'}, - {'compiler': 'pgi'}, - {'compiler': 'intel'}] - run_sys_tests(machine=machine, cime_path=self._cime_path(), - suite_name='my_suite') + mock_get_tests.return_value = [ + {"compiler": "intel"}, + {"compiler": "pgi"}, + {"compiler": "intel"}, + ] + run_sys_tests( + machine=machine, cime_path=self._cime_path(), suite_name="my_suite" + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 2) for command in all_commands: - six.assertRegex(self, command.cmd, - r'--xml-category +{}(\s|$)'.format('my_suite')) - six.assertRegex(self, command.cmd, - r'--xml-machine +{}(\s|$)'.format(self._MACHINE_NAME)) - - six.assertRegex(self, all_commands[0].cmd, r'--xml-compiler +intel(\s|$)') - six.assertRegex(self, all_commands[1].cmd, r'--xml-compiler +pgi(\s|$)') - - expected_testid1 = '{}_int'.format(self._expected_testid()) - expected_testid2 = '{}_pgi'.format(self._expected_testid()) - six.assertRegex(self, all_commands[0].cmd, - r'--test-id +{}(\s|$)'.format(expected_testid1)) - six.assertRegex(self, all_commands[1].cmd, - r'--test-id +{}(\s|$)'.format(expected_testid2)) + six.assertRegex( + self, command.cmd, r"--xml-category +{}(\s|$)".format("my_suite") + ) + six.assertRegex( + self, command.cmd, r"--xml-machine +{}(\s|$)".format(self._MACHINE_NAME) + ) + + six.assertRegex(self, all_commands[0].cmd, r"--xml-compiler +intel(\s|$)") + six.assertRegex(self, all_commands[1].cmd, r"--xml-compiler +pgi(\s|$)") + + expected_testid1 = "{}_int".format(self._expected_testid()) + expected_testid2 = "{}_pgi".format(self._expected_testid()) + six.assertRegex( + self, all_commands[0].cmd, r"--test-id +{}(\s|$)".format(expected_testid1) + ) + six.assertRegex( + self, all_commands[1].cmd, r"--test-id +{}(\s|$)".format(expected_testid2) + ) expected_testroot_path = os.path.join(self._scratch, self._expected_testroot()) - self.assertEqual(all_commands[0].out, os.path.join(expected_testroot_path, - 'STDOUT.'+expected_testid1)) - self.assertEqual(all_commands[0].err, os.path.join(expected_testroot_path, - 'STDERR.'+expected_testid1)) - self.assertEqual(all_commands[1].out, os.path.join(expected_testroot_path, - 'STDOUT.'+expected_testid2)) - self.assertEqual(all_commands[1].err, os.path.join(expected_testroot_path, - 'STDERR.'+expected_testid2)) - - expected_cs_status = os.path.join(self._scratch, - self._expected_testroot(), - 'cs.status') - expected_cs_status = os.path.join(self._scratch, - self._expected_testroot(), - 'cs.status.fails') + self.assertEqual( + all_commands[0].out, + os.path.join(expected_testroot_path, "STDOUT." + expected_testid1), + ) + self.assertEqual( + all_commands[0].err, + os.path.join(expected_testroot_path, "STDERR." + expected_testid1), + ) + self.assertEqual( + all_commands[1].out, + os.path.join(expected_testroot_path, "STDOUT." + expected_testid2), + ) + self.assertEqual( + all_commands[1].err, + os.path.join(expected_testroot_path, "STDERR." + expected_testid2), + ) + + expected_cs_status = os.path.join( + self._scratch, self._expected_testroot(), "cs.status" + ) + expected_cs_status = os.path.join( + self._scratch, self._expected_testroot(), "cs.status.fails" + ) self.assertTrue(os.path.isfile(expected_cs_status)) def test_createTestCommands_testsuiteSpecifiedCompilers(self): """The correct commands should be run with a test suite where compilers are specified""" machine = self._make_machine() - with mock.patch('ctsm.run_sys_tests.get_tests_from_xml') as mock_get_tests: + with mock.patch("ctsm.run_sys_tests.get_tests_from_xml") as mock_get_tests: # This value should be ignored; we just set it to make sure it's different # from the passed-in compiler list - mock_get_tests.return_value = [{'compiler': 'intel'}, - {'compiler': 'pgi'}, - {'compiler': 'gnu'}] - run_sys_tests(machine=machine, cime_path=self._cime_path(), - suite_name='my_suite', - suite_compilers=['comp1a', 'comp2b']) + mock_get_tests.return_value = [ + {"compiler": "intel"}, + {"compiler": "pgi"}, + {"compiler": "gnu"}, + ] + run_sys_tests( + machine=machine, + cime_path=self._cime_path(), + suite_name="my_suite", + suite_compilers=["comp1a", "comp2b"], + ) all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 2) - six.assertRegex(self, all_commands[0].cmd, r'--xml-compiler +comp1a(\s|$)') - six.assertRegex(self, all_commands[1].cmd, r'--xml-compiler +comp2b(\s|$)') + six.assertRegex(self, all_commands[0].cmd, r"--xml-compiler +comp1a(\s|$)") + six.assertRegex(self, all_commands[1].cmd, r"--xml-compiler +comp2b(\s|$)") def test_withDryRun_nothingDone(self): """With dry_run=True, no directories should be created, and no commands should be run""" machine = self._make_machine() - run_sys_tests(machine=machine, cime_path=self._cime_path(), testlist=['foo'], - dry_run=True) + run_sys_tests( + machine=machine, cime_path=self._cime_path(), testlist=["foo"], dry_run=True + ) self.assertEqual(os.listdir(self._scratch), []) self.assertEqual(machine.job_launcher.get_commands(), []) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index cad2a7d1af..a4ad42a9e8 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -16,6 +16,7 @@ # to make readable unit test names # pylint: disable=invalid-name + class TestUtilsFillTemplateFile(unittest.TestCase): """Tests of utils: fill_template_file""" @@ -27,19 +28,18 @@ def tearDown(self): def test_fillTemplateFile_basic(self): """Basic test of fill_template_file""" - template_path = os.path.join(self._testdir, 'template.txt') - final_path = os.path.join(self._testdir, 'final.txt') + template_path = os.path.join(self._testdir, "template.txt") + final_path = os.path.join(self._testdir, "final.txt") template_contents = """\ Hello $foo Goodbye $bar """ - with open(template_path, 'w') as f: + with open(template_path, "w") as f: f.write(template_contents) - fillins = {'foo':'aardvark', - 'bar':'zyzzyva'} + fillins = {"foo": "aardvark", "bar": "zyzzyva"} fill_template_file(template_path, final_path, fillins) expected_final_text = """\ @@ -93,8 +93,9 @@ def test_lonRange0To360_outOfBounds(self): """ Tests that lon_range_0_to_360 aborts gracefully when lon = 361 """ - with self.assertRaisesRegex(SystemExit, - "lon_in needs to be in the range 0 to 360"): + with self.assertRaisesRegex( + SystemExit, "lon_in needs to be in the range 0 to 360" + ): _ = lon_range_0_to_360(361) @@ -106,36 +107,48 @@ def test_handleConfigValue_UnsetCantBeUnset(self): Tests the handling of UNSET variable read in from a .cfg file for which can_be_unset = False """ - val = 'UNSET' - item = 'varname_in_cfg_file' + val = "UNSET" + item = "varname_in_cfg_file" default = None is_list = False convert_to_type = None can_be_unset = False allowed_values = None - errmsg = 'Must set a value for .cfg file variable: {}'.format(item) + errmsg = "Must set a value for .cfg file variable: {}".format(item) with self.assertRaisesRegex(SystemExit, errmsg): - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) def test_handleConfigValue_UnsetCanBeUnset(self): """ Tests the handling of UNSET variable read in from a .cfg file for which can_be_unset = True """ - val = 'UNSET' - item = 'varname_in_cfg_file' + val = "UNSET" + item = "varname_in_cfg_file" default = [True, False, True] is_list = True convert_to_type = None can_be_unset = True allowed_values = None - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) self.assertEqual(val, default) @@ -144,36 +157,48 @@ def test_handleConfigValue_convertToBoolFail(self): Tests the handling of misspelled boolean read in from a .cfg file Also test whether the code can read a list of booleans """ - val = 'False Tree False' # intentionally misspelled True - item = 'varname_in_cfg_file' + val = "False Tree False" # intentionally misspelled True + item = "varname_in_cfg_file" default = None is_list = True convert_to_type = bool can_be_unset = False allowed_values = None - errmsg = 'Non-boolean value found for .cfg file variable: {}'.format(item) + errmsg = "Non-boolean value found for .cfg file variable: {}".format(item) with self.assertRaisesRegex(SystemExit, errmsg): - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) def test_handleConfigValue_convertToBoolPass(self): """ Tests the handling of boolean read in from a .cfg file Also test whether the code can read a list of booleans """ - val = 'yes no' - item = 'varname_in_cfg_file' + val = "yes no" + item = "varname_in_cfg_file" default = None is_list = True convert_to_type = bool can_be_unset = False allowed_values = None - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) self.assertTrue(val[0]) self.assertFalse(val[1]) @@ -182,17 +207,23 @@ def test_handleConfigValue_convertToTypePass(self): """ Tests the handling of non-boolean list from a .cfg file """ - val = '-9 0.001' - item = 'varname_in_cfg_file' + val = "-9 0.001" + item = "varname_in_cfg_file" default = None is_list = True convert_to_type = float can_be_unset = False allowed_values = None - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) self.assertEqual(val[0], -9) self.assertEqual(val[1], 0.001) @@ -201,76 +232,102 @@ def test_handleConfigValue_convertToTypeFail(self): """ Tests the handling of an incorrectly entered list from a .cfg file """ - val = '1 2 3 x 5 6 7' - item = 'varname_in_cfg_file' + val = "1 2 3 x 5 6 7" + item = "varname_in_cfg_file" default = None is_list = True convert_to_type = float can_be_unset = False allowed_values = None - errmsg = 'Wrong type for .cfg file variable: {}'.format(item) + errmsg = "Wrong type for .cfg file variable: {}".format(item) with self.assertRaisesRegex(SystemExit, errmsg): - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) def test_handleConfigValue_allowedValsFail(self): """ Tests that the code aborts if val does not include all allowed_values """ - val = '1 2 3 4.5 6 7' - item = 'varname_in_cfg_file' + val = "1 2 3 4.5 6 7" + item = "varname_in_cfg_file" default = None is_list = True convert_to_type = float can_be_unset = False allowed_values = [1, 2, 3, 4, 5, 6, 7] v = 4.5 # v must equal the misstyped value in val - errmsg = ('{} is not an allowed value for {} in .cfg file. Check allowed_values'. - format(v, item)) + errmsg = "{} is not an allowed value for {} in .cfg file. Check allowed_values".format( + v, item + ) with self.assertRaisesRegex(SystemExit, errmsg): - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) def test_handleConfigValue_isListFail(self): """ Tests that the code aborts if we forget to set is_list = True """ - val = 'True False' - item = 'varname_in_cfg_file' + val = "True False" + item = "varname_in_cfg_file" default = None is_list = False convert_to_type = bool can_be_unset = False allowed_values = None - errmsg = 'More than 1 element found for .cfg file variable: {}'.format(item) + errmsg = "More than 1 element found for .cfg file variable: {}".format(item) with self.assertRaisesRegex(SystemExit, errmsg): - val = _handle_config_value(var=val, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val = _handle_config_value( + var=val, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) def test_handleConfigValue_isListFalse(self): """ Tests that the code works for a basic case of is_list = False """ - val_in = '0.5' - item = 'varname_in_cfg_file' + val_in = "0.5" + item = "varname_in_cfg_file" default = None is_list = False convert_to_type = float can_be_unset = False allowed_values = None - val_out = _handle_config_value(var=val_in, default=default, item=item, - is_list=is_list, convert_to_type=convert_to_type, - can_be_unset=can_be_unset, allowed_values=allowed_values) + val_out = _handle_config_value( + var=val_in, + default=default, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=can_be_unset, + allowed_values=allowed_values, + ) self.assertEqual(val_out, float(val_in)) -if __name__ == '__main__': + +if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/toolchain/ctsm_case.py b/python/ctsm/toolchain/ctsm_case.py index fe077ef3aa..2e2a5544f1 100755 --- a/python/ctsm/toolchain/ctsm_case.py +++ b/python/ctsm/toolchain/ctsm_case.py @@ -18,6 +18,7 @@ # -- import local classes for this script logger = logging.getLogger(__name__) + class CtsmCase: """ A class to encapsulate different ctsm cases. @@ -69,8 +70,8 @@ class CtsmCase: Build the namelist/control file for a ctsm case. """ - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes def __init__( self, @@ -83,7 +84,7 @@ def __init__( glc_flag, start_year, end_year, - hres_flag + hres_flag, ): self.res = res self.glc_nec = glc_nec @@ -96,10 +97,9 @@ def __init__( self.end_year = end_year self.hres_flag = hres_flag self.lu_fname = None - self.namelist_fname =None - self.ssp_val=None - self.rcp_val=None - + self.namelist_fname = None + self.ssp_val = None + self.rcp_val = None # -- check if end year value is a valid value self.check_endyear() @@ -152,8 +152,9 @@ def check_num_pft(self): self.num_pft = "78" else: self.num_pft = "16" - logger.debug(" crop_flag = %s => num_pft = %i", - self.crop_flag.__str__(), self.num_pft) + logger.debug( + " crop_flag = %s => num_pft = %i", self.crop_flag.__str__(), self.num_pft + ) def build_landuse_filename(self): """ @@ -179,7 +180,7 @@ def create_landuse_file(self): Create land-use txt file for a transient case. """ self.build_landuse_filename() - with open(self.lu_fname, "w", encoding='utf-8') as lu_file: + with open(self.lu_fname, "w", encoding="utf-8") as lu_file: for year in range(self.start_year, self.end_year + 1): @@ -217,7 +218,9 @@ def create_landuse_file(self): # -- Check if the land-use input file exist: if not os.path.isfile(lu_input_fname): logger.debug("lu_input_fname: %s", lu_input_fname) - logger.warning("land-use input file does not exist for year: %i.", year) + logger.warning( + "land-use input file does not exist for year: %i.", year + ) # TODO: make the space/tab exactly the same as pl code: lu_line = lu_input_fname + "\t\t\t" + str(year) + "\n" @@ -229,7 +232,7 @@ def create_landuse_file(self): logger.debug("year : %s", year) logger.debug(lu_line) - print ("Successfully created land use file : ", self.lu_fname, ".") + print("Successfully created land use file : ", self.lu_fname, ".") print("-------------------------------------------------------") def build_namelist_filename(self): @@ -265,7 +268,7 @@ def create_namelist_file(self): self.create_landuse_file() self.build_namelist_filename() - with open(self.namelist_fname, "w", encoding='utf-8') as namelist_file: + with open(self.namelist_fname, "w", encoding="utf-8") as namelist_file: label = get_ctsm_git_describe() @@ -295,7 +298,10 @@ def create_namelist_file(self): + self.input_path + "mksrf_LakePnDepth_3x3min_simyr2004_csplk_c151015.nc" + "\n" - "mksrf_fwetlnd = " + self.input_path + "mksrf_lanwat.050425.nc" + "\n" + "mksrf_fwetlnd = " + + self.input_path + + "mksrf_lanwat.050425.nc" + + "\n" "mksrf_fmax = " + self.input_path + "mksrf_fmax_3x3min_USGS_c120911.nc" diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index dd30c3fb31..518b8077b7 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -40,6 +40,7 @@ def fill_template_file(path_to_template, path_to_final, substitutions): with open(path_to_final, "w") as final_file: final_file.write(final_file_contents) + def add_tag_to_filename(filename, tag): """ Add a tag and replace timetag of a filename diff --git a/python/six.py b/python/six.py index df4958f3b8..b5b008a2dc 100644 --- a/python/six.py +++ b/python/six.py @@ -738,7 +738,6 @@ def reraise(tp, value, tb=None): value = None tb = None - else: def exec_(_code_, _globs_=None, _locs_=None): @@ -877,7 +876,6 @@ def wrapper(f): return wrapper - else: wraps = functools.wraps From 1ec921007ac7f9660ebf8c2e347d9ee7b3f5303a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Mar 2022 17:33:30 -0700 Subject: [PATCH 07/16] Add another black reformat --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 517bd4cc8c..62cd314d46 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,3 @@ # Ran python directory through black python formatter +0aa2957c1f8603c63fa30b11295c06cfddff44a5 2cdb380febb274478e84cd90945aee93f29fa2e6 From 4cd83cb3ee6d85eb909403487abf5eeaf4d98911 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 6 Jun 2022 12:29:42 -0600 Subject: [PATCH 08/16] Black update --- .github/workflows/black.yml | 2 +- python/ctsm/args_utils.py | 4 +- python/ctsm/config_utils.py | 8 +-- python/ctsm/git_utils.py | 10 +-- python/ctsm/joblauncher/job_launcher_base.py | 4 +- .../ctsm/joblauncher/job_launcher_factory.py | 4 +- python/ctsm/joblauncher/job_launcher_fake.py | 4 +- python/ctsm/lilac_build_ctsm.py | 36 +++------- python/ctsm/lilac_download_input_data.py | 4 +- python/ctsm/lilac_make_runtime_inputs.py | 20 ++---- python/ctsm/machine.py | 6 +- python/ctsm/machine_defaults.py | 8 +-- .../ctsm/modify_fsurdat/fsurdat_modifier.py | 8 +-- python/ctsm/modify_fsurdat/modify_fsurdat.py | 16 ++--- python/ctsm/os_utils.py | 4 +- python/ctsm/path_utils.py | 2 +- python/ctsm/run_ctsm_py_tests.py | 8 +-- python/ctsm/run_sys_tests.py | 39 +++-------- .../ctsm/site_and_regional/regional_case.py | 4 +- .../site_and_regional/single_point_case.py | 68 ++++++------------- python/ctsm/subset_data.py | 16 ++--- python/ctsm/test/test_sys_fsurdat_modifier.py | 12 +--- python/ctsm/test/test_sys_lilac_build_ctsm.py | 8 +-- python/ctsm/test/test_unit_args_utils.py | 20 ++---- .../ctsm/test/test_unit_lilac_build_ctsm.py | 20 ++---- .../test_unit_lilac_make_runtime_inputs.py | 4 +- python/ctsm/test/test_unit_machine.py | 12 +--- python/ctsm/test/test_unit_modify_fsurdat.py | 4 +- python/ctsm/test/test_unit_run_sys_tests.py | 44 +++--------- python/ctsm/test/test_unit_singlept_data.py | 12 +--- .../test/test_unit_singlept_data_surfdata.py | 43 ++++++------ python/ctsm/test/test_unit_utils.py | 4 +- python/ctsm/toolchain/ctsm_case.py | 36 +++------- python/six.py | 47 +++---------- 34 files changed, 153 insertions(+), 388 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 5c8eef7254..d351f04817 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -13,7 +13,7 @@ jobs: # Use a specific version of the black-checker # Use version that's the same as with conda on cheyenne # Tag for black-check corresponds to the black version tag - - uses: jpetrucciani/black-check@21.11b1 + - uses: jpetrucciani/black-check@21.12b0 with: path: python black_flags: --config python/pyproject.toml diff --git a/python/ctsm/args_utils.py b/python/ctsm/args_utils.py index 357d6a41da..5029c01f55 100644 --- a/python/ctsm/args_utils.py +++ b/python/ctsm/args_utils.py @@ -27,9 +27,7 @@ def plat_type(plat): """ plat_out = float(plat) if plat_out < -90 or plat_out > 90: - raise argparse.ArgumentTypeError( - "ERROR: Latitude should be between -90 and 90." - ) + raise argparse.ArgumentTypeError("ERROR: Latitude should be between -90 and 90.") return plat_out diff --git a/python/ctsm/config_utils.py b/python/ctsm/config_utils.py index 0ece3a180a..847a1804f6 100644 --- a/python/ctsm/config_utils.py +++ b/python/ctsm/config_utils.py @@ -63,9 +63,7 @@ def get_config_value( try: val = config.get(section, item) except configparser.NoSectionError: - abort( - "ERROR: Config file {} must contain section '{}'".format(file_path, section) - ) + abort("ERROR: Config file {} must contain section '{}'".format(file_path, section)) except configparser.NoOptionError: abort( "ERROR: Config file {} must contain item '{}' in section '{}'".format( @@ -74,9 +72,7 @@ def get_config_value( ) if val == _CONFIG_PLACEHOLDER: - abort( - "Error: {} needs to be specified in config file {}".format(item, file_path) - ) + abort("Error: {} needs to be specified in config file {}".format(item, file_path)) val = _handle_config_value( var=val, diff --git a/python/ctsm/git_utils.py b/python/ctsm/git_utils.py index 65648a2fbf..658788dbc2 100644 --- a/python/ctsm/git_utils.py +++ b/python/ctsm/git_utils.py @@ -20,9 +20,7 @@ def get_ctsm_git_short_hash(): sha (str) : git short hash for ctsm repository """ sha = ( - subprocess.check_output( - ["git", "-C", path_to_ctsm_root(), "rev-parse", "--short", "HEAD"] - ) + subprocess.check_output(["git", "-C", path_to_ctsm_root(), "rev-parse", "--short", "HEAD"]) .strip() .decode() ) @@ -59,9 +57,5 @@ def get_ctsm_git_describe(): Returns: label (str) : ouput of running 'git describe' for the CTSM repository """ - label = ( - subprocess.check_output(["git", "-C", path_to_ctsm_root(), "describe"]) - .strip() - .decode() - ) + label = subprocess.check_output(["git", "-C", path_to_ctsm_root(), "describe"]).strip().decode() return label diff --git a/python/ctsm/joblauncher/job_launcher_base.py b/python/ctsm/joblauncher/job_launcher_base.py index 86075aa27e..cfad9cce83 100644 --- a/python/ctsm/joblauncher/job_launcher_base.py +++ b/python/ctsm/joblauncher/job_launcher_base.py @@ -91,9 +91,7 @@ def run_command(self, command, stdout_path, stderr_path, dry_run=False): If dry_run is True, then just print the command to be run without actually running it. """ - logger.info( - "%s", self.run_command_logger_message(command, stdout_path, stderr_path) - ) + logger.info("%s", self.run_command_logger_message(command, stdout_path, stderr_path)) if not dry_run: self.run_command_impl(command, stdout_path, stderr_path) diff --git a/python/ctsm/joblauncher/job_launcher_factory.py b/python/ctsm/joblauncher/job_launcher_factory.py index 9a615769fd..39e4ddb5b1 100644 --- a/python/ctsm/joblauncher/job_launcher_factory.py +++ b/python/ctsm/joblauncher/job_launcher_factory.py @@ -75,7 +75,5 @@ def _assert_not_none(arg, arg_name, job_launcher_type): """Raises an exception if the given argument has value None""" if arg is None: raise TypeError( - "{} cannot be None for job launcher of type {}".format( - arg_name, job_launcher_type - ) + "{} cannot be None for job launcher of type {}".format(arg_name, job_launcher_type) ) diff --git a/python/ctsm/joblauncher/job_launcher_fake.py b/python/ctsm/joblauncher/job_launcher_fake.py index 8c52bf7688..ff73b6d3a3 100644 --- a/python/ctsm/joblauncher/job_launcher_fake.py +++ b/python/ctsm/joblauncher/job_launcher_fake.py @@ -17,9 +17,7 @@ def __init__(self): self._commands = [] def run_command_impl(self, command, stdout_path, stderr_path): - self._commands.append( - Command(cmd=" ".join(command), out=stdout_path, err=stderr_path) - ) + self._commands.append(Command(cmd=" ".join(command), out=stdout_path, err=stderr_path)) def run_command_logger_message(self, command, stdout_path, stderr_path): message = ( diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index 99e09d0614..20231c0df9 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -31,13 +31,9 @@ _PATH_TO_TEMPLATES = os.path.join(path_to_ctsm_root(), "lilac", "bld_templates") -_PATH_TO_MAKE_RUNTIME_INPUTS = os.path.join( - path_to_ctsm_root(), "lilac", "make_runtime_inputs" -) +_PATH_TO_MAKE_RUNTIME_INPUTS = os.path.join(path_to_ctsm_root(), "lilac", "make_runtime_inputs") -_PATH_TO_DOWNLOAD_INPUT_DATA = os.path.join( - path_to_ctsm_root(), "lilac", "download_input_data" -) +_PATH_TO_DOWNLOAD_INPUT_DATA = os.path.join(path_to_ctsm_root(), "lilac", "download_input_data") _MACHINE_CONFIG_DIRNAME = "machine_configuration" _INPUTDATA_DIRNAME = "inputdata" @@ -160,9 +156,7 @@ def build_ctsm( if machine is None: assert os_type is not None, "with machine absent, os_type must be given" assert netcdf_path is not None, "with machine absent, netcdf_path must be given" - assert ( - esmf_mkfile_path is not None - ), "with machine absent, esmf_mkfile_path must be given" + assert esmf_mkfile_path is not None, "with machine absent, esmf_mkfile_path must be given" assert max_mpitasks_per_node is not None, ( "with machine absent " "max_mpitasks_per_node must be given" ) @@ -230,9 +224,7 @@ def rebuild_ctsm(build_dir): cwd=case_dir, ) except subprocess.CalledProcessError: - abort( - "ERROR resetting build for CTSM in order to rebuild - see above for details" - ) + abort("ERROR resetting build for CTSM in order to rebuild - see above for details") _build_case(build_dir) @@ -627,12 +619,8 @@ def _fill_out_machine_files( # ------------------------------------------------------------------------ fill_template_file( - path_to_template=os.path.join( - _PATH_TO_TEMPLATES, "config_machines_template.xml" - ), - path_to_final=os.path.join( - build_dir, _MACHINE_CONFIG_DIRNAME, "config_machines.xml" - ), + path_to_template=os.path.join(_PATH_TO_TEMPLATES, "config_machines_template.xml"), + path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "config_machines.xml"), substitutions={ "OS": os_type, "COMPILER": compiler, @@ -753,9 +741,7 @@ def _create_case( create_newcase_cmd.extend(machine_args) if inputdata_path: create_newcase_cmd.extend(["--input-dir", inputdata_path]) - run_cmd_output_on_error( - create_newcase_cmd, errmsg="Problem creating CTSM case directory" - ) + run_cmd_output_on_error(create_newcase_cmd, errmsg="Problem creating CTSM case directory") subprocess.check_call([xmlchange, "LILAC_MODE=on"], cwd=case_dir) if build_debug: @@ -814,9 +800,7 @@ def _stage_runtime_inputs(build_dir, no_pnetcdf): pio_rearranger = 2 fill_template_file( path_to_template=os.path.join(_PATH_TO_TEMPLATES, "lnd_modelio_template.nml"), - path_to_final=os.path.join( - build_dir, _RUNTIME_INPUTS_DIRNAME, "lnd_modelio.nml" - ), + path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, "lnd_modelio.nml"), substitutions={ "PIO_REARRANGER": pio_rearranger, "PIO_STRIDE": pio_stride, @@ -861,9 +845,7 @@ def _build_case(build_dir): except subprocess.CalledProcessError: abort("ERROR building CTSM or its dependencies - see above for details") - make_link( - os.path.join(case_dir, "bld", "ctsm.mk"), os.path.join(build_dir, "ctsm.mk") - ) + make_link(os.path.join(case_dir, "bld", "ctsm.mk"), os.path.join(build_dir, "ctsm.mk")) def _xmlquery(varname, build_dir): diff --git a/python/ctsm/lilac_download_input_data.py b/python/ctsm/lilac_download_input_data.py index 90ea5ce23f..a70b64d134 100644 --- a/python/ctsm/lilac_download_input_data.py +++ b/python/ctsm/lilac_download_input_data.py @@ -84,9 +84,7 @@ def _commandline_args(): def _create_lilac_input_data_list(rundir): with open(os.path.join(rundir, "lilac_in")) as lilac_in: - with open( - os.path.join(rundir, "lilac.input_data_list"), "w" - ) as input_data_list: + with open(os.path.join(rundir, "lilac.input_data_list"), "w") as input_data_list: for line in lilac_in: if re.search(_LILAC_FILENAME, line): # Remove quotes from filename, then output this line diff --git a/python/ctsm/lilac_make_runtime_inputs.py b/python/ctsm/lilac_make_runtime_inputs.py index f9f8dcd9b4..15dc4dd5b9 100644 --- a/python/ctsm/lilac_make_runtime_inputs.py +++ b/python/ctsm/lilac_make_runtime_inputs.py @@ -118,16 +118,12 @@ def determine_bldnml_opts(bgc_mode, crop, vichydro): if crop == "on": if bgc_mode not in ["bgc", "cn"]: - abort( - "Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'" - ) + abort("Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'") bldnml_opts += " -crop" if vichydro == "on": if bgc_mode != "sp": - abort( - "Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'" - ) + abort("Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'") bldnml_opts += " -vichydro" return bldnml_opts @@ -148,9 +144,7 @@ def buildnml(cime_path, rundir): config = ConfigParser() config.read(ctsm_cfg_path) - lnd_domain_file = get_config_value( - config, "buildnml_input", "lnd_domain_file", ctsm_cfg_path - ) + lnd_domain_file = get_config_value(config, "buildnml_input", "lnd_domain_file", ctsm_cfg_path) fsurdat = get_config_value( config, "buildnml_input", "fsurdat", ctsm_cfg_path, can_be_unset=True ) @@ -201,16 +195,12 @@ def buildnml(cime_path, rundir): co2_ppmv = get_config_value(config, "buildnml_input", "co2_ppmv", ctsm_cfg_path) use_case = get_config_value(config, "buildnml_input", "use_case", ctsm_cfg_path) - lnd_tuning_mode = get_config_value( - config, "buildnml_input", "lnd_tuning_mode", ctsm_cfg_path - ) + lnd_tuning_mode = get_config_value(config, "buildnml_input", "lnd_tuning_mode", ctsm_cfg_path) spinup = get_config_value( config, "buildnml_input", "spinup", ctsm_cfg_path, allowed_values=["off", "on"] ) - inputdata_path = get_config_value( - config, "buildnml_input", "inputdata_path", ctsm_cfg_path - ) + inputdata_path = get_config_value(config, "buildnml_input", "inputdata_path", ctsm_cfg_path) # Parse the user_nl_ctsm file infile = os.path.join(rundir, ".namelist") diff --git a/python/ctsm/machine.py b/python/ctsm/machine.py index fec7908cb0..18056063c1 100644 --- a/python/ctsm/machine.py +++ b/python/ctsm/machine.py @@ -122,11 +122,7 @@ def create_machine( # in a particular case. create_test_retry = mach_defaults.create_test_retry create_test_queue = mach_defaults.create_test_queue - if ( - account is None - and mach_defaults.account_required - and not allow_missing_entries - ): + if account is None and mach_defaults.account_required and not allow_missing_entries: raise RuntimeError("Could not find an account code") else: if not allow_missing_entries: diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index e31ea9584d..7486237323 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -41,17 +41,13 @@ # have defaults for qsub, because other launchers (like no_batch) don't need any # arguments. -QsubDefaults = namedtuple( - "QsubDefaults", ["queue", "walltime", "extra_args", "required_args"] -) +QsubDefaults = namedtuple("QsubDefaults", ["queue", "walltime", "extra_args", "required_args"]) MACHINE_DEFAULTS = { "cheyenne": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), - baseline_dir=os.path.join( - os.path.sep, "glade", "p", "cgd", "tss", "ctsm_baselines" - ), + baseline_dir=os.path.join(os.path.sep, "glade", "p", "cgd", "tss", "ctsm_baselines"), account_required=True, create_test_retry=0, # NOTE(wjs, 2022-02-23) By default, use the regular queue, even for diff --git a/python/ctsm/modify_fsurdat/fsurdat_modifier.py b/python/ctsm/modify_fsurdat/fsurdat_modifier.py index 18e11cf2df..43b10b1c0b 100644 --- a/python/ctsm/modify_fsurdat/fsurdat_modifier.py +++ b/python/ctsm/modify_fsurdat/fsurdat_modifier.py @@ -31,9 +31,7 @@ def main(): # read the command line argument to obtain the path to the .cfg file parser = argparse.ArgumentParser() - parser.add_argument( - "cfg_path", help="/path/name.cfg of input file, eg ./modify.cfg" - ) + parser.add_argument("cfg_path", help="/path/name.cfg of input file, eg ./modify.cfg") add_logging_args(parser) args = parser.parse_args() process_logging_args(args) @@ -203,9 +201,7 @@ def fsurdat_modifier(cfg_path): if idealized: modify_fsurdat.set_idealized() # set 2D variables # set 3D and 4D variables pertaining to natural vegetation - modify_fsurdat.set_dom_plant( - dom_plant=0, lai=[], sai=[], hgt_top=[], hgt_bot=[] - ) + modify_fsurdat.set_dom_plant(dom_plant=0, lai=[], sai=[], hgt_top=[], hgt_bot=[]) logger.info("idealized complete") if max_sat_area is not None: # overwrite "idealized" value diff --git a/python/ctsm/modify_fsurdat/modify_fsurdat.py b/python/ctsm/modify_fsurdat/modify_fsurdat.py index 3e7b267505..5864f5f50f 100644 --- a/python/ctsm/modify_fsurdat/modify_fsurdat.py +++ b/python/ctsm/modify_fsurdat/modify_fsurdat.py @@ -162,9 +162,9 @@ def set_dom_plant(self, dom_plant, lai, sai, hgt_top, hgt_bot): # If dom_plant is a cft, add PCT_NATVEG to PCT_CROP in the rectangle # and remove same from PCT_NATVEG, i.e. set PCT_NATVEG = 0. if dom_plant > max(self.file.natpft): # dom_plant is a cft (crop) - self.file["PCT_CROP"] = self.file["PCT_CROP"] + self.file[ - "PCT_NATVEG" - ].where(self.rectangle, other=0) + self.file["PCT_CROP"] = self.file["PCT_CROP"] + self.file["PCT_NATVEG"].where( + self.rectangle, other=0 + ) self.setvar_lev0("PCT_NATVEG", 0) for cft in self.file.cft: @@ -173,9 +173,7 @@ def set_dom_plant(self, dom_plant, lai, sai, hgt_top, hgt_bot): self.setvar_lev1("PCT_CFT", val=0, lev1_dim=cft_local) # set 3D variable - self.setvar_lev1( - "PCT_CFT", val=100, lev1_dim=dom_plant - (max(self.file.natpft) + 1) - ) + self.setvar_lev1("PCT_CFT", val=100, lev1_dim=dom_plant - (max(self.file.natpft) + 1)) else: # dom_plant is a pft (not a crop) for pft in self.file.natpft: # initialize 3D variable; set outside the loop below @@ -252,9 +250,9 @@ def setvar_lev2(self, var, val, lev1_dim, lev2_dim): Sets 4d variable var to value val in user-defined rectangle, defined as "other" in the function """ - self.file[var][lev2_dim, lev1_dim, ...] = self.file[var][ - lev2_dim, lev1_dim, ... - ].where(self.not_rectangle, other=val) + self.file[var][lev2_dim, lev1_dim, ...] = self.file[var][lev2_dim, lev1_dim, ...].where( + self.not_rectangle, other=val + ) def set_idealized(self): """ diff --git a/python/ctsm/os_utils.py b/python/ctsm/os_utils.py index 8dad6136ca..b9b926c816 100644 --- a/python/ctsm/os_utils.py +++ b/python/ctsm/os_utils.py @@ -18,9 +18,7 @@ def run_cmd_output_on_error(cmd, errmsg, cwd=None): cwd: string or None - path from which the command should be run """ try: - _ = subprocess.check_output( - cmd, stderr=subprocess.STDOUT, universal_newlines=True, cwd=cwd - ) + _ = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True, cwd=cwd) except subprocess.CalledProcessError as error: print("ERROR while running:") print(" ".join(cmd)) diff --git a/python/ctsm/path_utils.py b/python/ctsm/path_utils.py index b81973f1b7..0015822c03 100644 --- a/python/ctsm/path_utils.py +++ b/python/ctsm/path_utils.py @@ -91,7 +91,7 @@ def add_cime_lib_to_path(standalone_only=False): """ cime_path = path_to_cime(standalone_only=standalone_only) prepend_to_python_path(cime_path) - cime_lib_path = os.path.join(cime_path, 'CIME', 'Tools') + cime_lib_path = os.path.join(cime_path, "CIME", "Tools") prepend_to_python_path(cime_lib_path) return cime_path diff --git a/python/ctsm/run_ctsm_py_tests.py b/python/ctsm/run_ctsm_py_tests.py index 91ce61a211..0542dc41cb 100644 --- a/python/ctsm/run_ctsm_py_tests.py +++ b/python/ctsm/run_ctsm_py_tests.py @@ -62,13 +62,9 @@ def _commandline_args(description): test_subset = parser.add_mutually_exclusive_group() - test_subset.add_argument( - "-u", "--unit", action="store_true", help="Only run unit tests" - ) + test_subset.add_argument("-u", "--unit", action="store_true", help="Only run unit tests") - test_subset.add_argument( - "-s", "--sys", action="store_true", help="Only run system tests" - ) + test_subset.add_argument("-s", "--sys", action="store_true", help="Only run system tests") test_subset.add_argument( "-p", "--pattern", help="File name pattern to match\n" "Default is test*.py" diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py index 54e85ca85d..992f2b544a 100644 --- a/python/ctsm/run_sys_tests.py +++ b/python/ctsm/run_sys_tests.py @@ -162,9 +162,7 @@ def run_sys_tests( + (testlist is not None and len(testlist) > 0) ) if num_provided_options != 1: - raise RuntimeError( - "Exactly one of suite_name, testfile or testlist must be provided" - ) + raise RuntimeError("Exactly one of suite_name, testfile or testlist must be provided") if testid_base is None: testid_base = _get_testid_base(machine.name) @@ -293,9 +291,7 @@ def _commandline_args(): tests_to_run.add_argument("-s", "--suite-name", help="Name of test suite to run") - tests_to_run.add_argument( - "-f", "--testfile", help="Path to file listing tests to run" - ) + tests_to_run.add_argument("-f", "--testfile", help="Path to file listing tests to run") tests_to_run.add_argument( "-t", @@ -325,13 +321,10 @@ def _commandline_args(): "-g", "--generate", metavar="GENERATE_NAME", - help="Baseline name (often tag) to generate\n" - "(required unless --skip-generate is given)", + help="Baseline name (often tag) to generate\n" "(required unless --skip-generate is given)", ) - generate.add_argument( - "--skip-generate", action="store_true", help="Do not generate baselines" - ) + generate.add_argument("--skip-generate", action="store_true", help="Do not generate baselines") parser.add_argument( "--suite-compiler", @@ -446,9 +439,7 @@ def _commandline_args(): help="Walltime for the create_test command.\n" "Only applies on machines for which we submit the create_test command\n" "rather than running it on the login node.\n" - "Default for this machine: {}".format( - default_machine.job_launcher.get_walltime() - ), + "Default for this machine: {}".format(default_machine.job_launcher.get_walltime()), ) parser.add_argument( @@ -457,9 +448,7 @@ def _commandline_args(): "create_test command.\n" "(To allow the argument parsing to accept this, enclose the string\n" 'in quotes, with a leading space, as in " --my-arg foo".)\n' - "Default for this machine: {}".format( - default_machine.job_launcher.get_extra_args() - ), + "Default for this machine: {}".format(default_machine.job_launcher.get_extra_args()), ) parser.add_argument( @@ -506,13 +495,9 @@ def _commandline_args(): def _check_arg_validity(args): if args.suite_compiler and not args.suite_name: - raise RuntimeError( - "--suite-compiler can only be specified if using --suite-name" - ) + raise RuntimeError("--suite-compiler can only be specified if using --suite-name") if args.rerun_existing_failures and not args.testid_base: - raise RuntimeError( - "With --rerun-existing-failures, must also specify --testid-base" - ) + raise RuntimeError("With --rerun-existing-failures, must also specify --testid-base") def _get_testid_base(machine_name): @@ -670,9 +655,7 @@ def _cs_status_xfail_arg(): expected fails xml file """ ctsm_root = path_to_ctsm_root() - xfail_path = os.path.join( - ctsm_root, "cime_config", "testdefs", "ExpectedTestFails.xml" - ) + xfail_path = os.path.join(ctsm_root, "cime_config", "testdefs", "ExpectedTestFails.xml") return "--expected-fails-file {}".format(xfail_path) @@ -720,9 +703,7 @@ def _get_compilers_for_suite(suite_name, machine_name): return compilers -def _run_create_test( - cime_path, test_args, machine, testid, testroot, create_test_args, dry_run -): +def _run_create_test(cime_path, test_args, machine, testid, testroot, create_test_args, dry_run): create_test_cmd = _build_create_test_cmd( cime_path=cime_path, test_args=test_args, diff --git a/python/ctsm/site_and_regional/regional_case.py b/python/ctsm/site_and_regional/regional_case.py index 57bb8474f4..1dc6a522cc 100644 --- a/python/ctsm/site_and_regional/regional_case.py +++ b/python/ctsm/site_and_regional/regional_case.py @@ -222,7 +222,5 @@ def create_landuse_at_reg(self, indir, file, user_mods_dir): if self.create_user_mods: with open(os.path.join(user_mods_dir, "user_nl_clm"), "a") as nl_clm: # line = "landuse = '${}'".format(os.path.join(USRDAT_DIR, fluse_out)) - line = "flanduse_timeseries = '${}'".format( - os.path.join(USRDAT_DIR, fluse_out) - ) + line = "flanduse_timeseries = '${}'".format(os.path.join(USRDAT_DIR, fluse_out)) self.write_to_file(line, nl_clm) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index a80615425a..92ea917669 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -136,7 +136,7 @@ def __init__( self.create_tag() self.check_dom_pft() - #self.check_nonveg() + # self.check_nonveg() self.check_pct_pft() def create_tag(self): @@ -303,12 +303,8 @@ def create_domain_at_point(self, indir, file): """ Create domain file for this SinglePointCase class. """ - logger.info( - "----------------------------------------------------------------------" - ) - logger.info( - "Creating domain file at %s, %s.", self.plon.__str__(), self.plat.__str__() - ) + logger.info("----------------------------------------------------------------------") + logger.info("Creating domain file at %s, %s.", self.plon.__str__(), self.plat.__str__()) # specify files fdomain_in = os.path.join(indir, file) @@ -339,9 +335,7 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): """ Create landuse file at a single point. """ - logger.info( - "----------------------------------------------------------------------" - ) + logger.info("----------------------------------------------------------------------") logger.info( "Creating land use file at %s, %s.", self.plon.__str__(), @@ -368,9 +362,7 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): # revert expand dimensions of YEAR year = np.squeeze(np.asarray(f_out["YEAR"])) - temp_xr = xr.DataArray( - year, coords={"time": f_out["time"]}, dims="time", name="YEAR" - ) + temp_xr = xr.DataArray(year, coords={"time": f_out["time"]}, dims="time", name="YEAR") temp_xr.attrs["units"] = "unitless" temp_xr.attrs["long_name"] = "Year of PFT data" f_out["YEAR"] = temp_xr @@ -388,9 +380,7 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): # write to user_nl_clm data if specified if self.create_user_mods: with open(os.path.join(user_mods_dir, "user_nl_clm"), "a") as nl_clm: - line = "flanduse_timeseries = '${}'".format( - os.path.join(USRDAT_DIR, fluse_out) - ) + line = "flanduse_timeseries = '${}'".format(os.path.join(USRDAT_DIR, fluse_out)) self.write_to_file(line, nl_clm) def modify_surfdata_atpoint(self, f_orig): @@ -443,11 +433,11 @@ def modify_surfdata_atpoint(self, f_orig): f_mod["PCT_NATVEG"][:, :] = 0 f_mod["PCT_CROP"][:, :] = 100 else: - #-- recalculate percentages after zeroing out non-veg landunits - #-- so they add up to 100%. - tot_pct = f_mod["PCT_CROP"]+f_mod['PCT_NATVEG'] - f_mod ["PCT_CROP"] = f_mod ["PCT_CROP"]/tot_pct * 100 - f_mod ["PCT_NATVEG"] = f_mod ["PCT_NATVEG"]/tot_pct * 100 + # -- recalculate percentages after zeroing out non-veg landunits + # -- so they add up to 100%. + tot_pct = f_mod["PCT_CROP"] + f_mod["PCT_NATVEG"] + f_mod["PCT_CROP"] = f_mod["PCT_CROP"] / tot_pct * 100 + f_mod["PCT_NATVEG"] = f_mod["PCT_NATVEG"] / tot_pct * 100 else: logger.info( @@ -467,9 +457,7 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir): Create surface data file at a single point. """ # pylint: disable=too-many-statements - logger.info( - "----------------------------------------------------------------------" - ) + logger.info("----------------------------------------------------------------------") logger.info( "Creating surface dataset file at %s, %s", self.plon.__str__(), @@ -538,9 +526,7 @@ def create_datmdomain_at_point(self, datm_tuple: DatmFiles): """ Create DATM domain file at a single point """ - logger.info( - "----------------------------------------------------------------------" - ) + logger.info("----------------------------------------------------------------------") logger.info( "Creating DATM domain file at %s, %s", self.plon.__str__(), @@ -604,12 +590,8 @@ def write_shell_commands(self, file): """ # write_to_file surrounds text with newlines with open(file, "w") as nl_file: - self.write_to_file( - "# Change below line if you move the subset data directory", nl_file - ) - self.write_to_file( - "./xmlchange {}={}".format(USRDAT_DIR, self.out_dir), nl_file - ) + self.write_to_file("# Change below line if you move the subset data directory", nl_file) + self.write_to_file("./xmlchange {}={}".format(USRDAT_DIR, self.out_dir), nl_file) self.write_to_file("./xmlchange PTS_LON={}".format(str(self.plon)), nl_file) self.write_to_file("./xmlchange PTS_LAT={}".format(str(self.plat)), nl_file) self.write_to_file("./xmlchange MPILIB=mpi-serial", nl_file) @@ -623,24 +605,16 @@ def write_datm_streams_lines(self, streamname, datmfiles, file): datmfiles - comma-separated list (str) of DATM file names file - file connection to user_nl_datm_streams file """ - self.write_to_file( - "{}:datafiles={}".format(streamname, ",".join(datmfiles)), file - ) + self.write_to_file("{}:datafiles={}".format(streamname, ",".join(datmfiles)), file) self.write_to_file("{}:mapalgo=none".format(streamname), file) self.write_to_file("{}:meshfile=none".format(streamname), file) - def create_datm_at_point( - self, datm_tuple: DatmFiles, datm_syr, datm_eyr, datm_streams_file - ): + def create_datm_at_point(self, datm_tuple: DatmFiles, datm_syr, datm_eyr, datm_streams_file): """ Create all of a DATM dataset at a point. """ - logger.info( - "----------------------------------------------------------------------" - ) - logger.info( - "Creating DATM files at %s, %s", self.plon.__str__(), self.plat.__str__() - ) + logger.info("----------------------------------------------------------------------") + logger.info("Creating DATM files at %s, %s", self.plon.__str__(), self.plat.__str__()) # -- create data files infile = [] @@ -689,9 +663,7 @@ def create_datm_at_point( precfiles.append( os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fprecip2) ) - tpqwfiles.append( - os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, ftpqw2) - ) + tpqwfiles.append(os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, ftpqw2)) for idx, out_f in enumerate(outfile): logger.debug(out_f) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 67f7090788..7a33d9c2fa 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -434,9 +434,7 @@ def subset_point(args, file_dict: dict): Subsets surface, domain, land use, and/or DATM files at a single point """ - logger.info( - "----------------------------------------------------------------------------" - ) + logger.info("----------------------------------------------------------------------------") logger.info("This script extracts a single point from the global CTSM datasets.") num_pft = int(determine_num_pft(args.crop_flag)) @@ -465,9 +463,7 @@ def subset_point(args, file_dict: dict): # -- Create CTSM domain file if single_point.create_domain: - single_point.create_domain_at_point( - file_dict["main_dir"], file_dict["fdomain_in"] - ) + single_point.create_domain_at_point(file_dict["main_dir"], file_dict["fdomain_in"]) # -- Create CTSM surface data file if single_point.create_surfdata: @@ -494,9 +490,7 @@ def subset_point(args, file_dict: dict): # -- Write shell commands if single_point.create_user_mods: - single_point.write_shell_commands( - os.path.join(args.user_mods_dir, "shell_commands") - ) + single_point.write_shell_commands(os.path.join(args.user_mods_dir, "shell_commands")) logger.info("Successfully ran script for single point.") @@ -506,9 +500,7 @@ def subset_region(args, file_dict: dict): Subsets surface, domain, land use, and/or DATM files for a region """ - logger.info( - "----------------------------------------------------------------------------" - ) + logger.info("----------------------------------------------------------------------------") logger.info("This script extracts a region from the global CTSM datasets.") # -- Create Region Object diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index c573fc7670..0be2305365 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -38,9 +38,7 @@ def setUp(self): self._cfg_template_path = os.path.join( path_to_ctsm_root(), "tools/modify_fsurdat/modify_template.cfg" ) - testinputs_path = os.path.join( - path_to_ctsm_root(), "python/ctsm/test/testinputs" - ) + testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") self._fsurdat_in = os.path.join( testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", @@ -93,9 +91,7 @@ def test_crop(self): self.assertFalse(fsurdat_out_data.equals(fsurdat_in_data)) # compare fsurdat_out to fsurdat_out_baseline located in /testinputs - fsurdat_out_baseline = ( - self._fsurdat_in[:-3] + "_modified_with_crop" + self._fsurdat_in[-3:] - ) + fsurdat_out_baseline = self._fsurdat_in[:-3] + "_modified_with_crop" + self._fsurdat_in[-3:] fsurdat_out_base_data = xr.open_dataset(fsurdat_out_baseline) # assert that fsurdat_out equals fsurdat_out_baseline self.assertTrue(fsurdat_out_data.equals(fsurdat_out_base_data)) @@ -120,9 +116,7 @@ def test_allInfo(self): self.assertFalse(fsurdat_out_data.equals(fsurdat_in_data)) # compare fsurdat_out to fsurdat_out_baseline located in /testinputs - fsurdat_out_baseline = ( - self._fsurdat_in[:-3] + "_modified" + self._fsurdat_in[-3:] - ) + fsurdat_out_baseline = self._fsurdat_in[:-3] + "_modified" + self._fsurdat_in[-3:] fsurdat_out_base_data = xr.open_dataset(fsurdat_out_baseline) # assert that fsurdat_out equals fsurdat_out_baseline self.assertTrue(fsurdat_out_data.equals(fsurdat_out_base_data)) diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index 3206a69522..f1c5e22f8f 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -34,16 +34,16 @@ def setUp(self): # user-defined machine infrastructure on an NCAR machine. So bypass this CIME # check by temporarily removing NCAR_HOST from the environment if it was present. # (Then we restore it in the tearDown method.) - if 'NCAR_HOST' in os.environ: - self._ncarhost = os.environ['NCAR_HOST'] - del os.environ['NCAR_HOST'] + if "NCAR_HOST" in os.environ: + self._ncarhost = os.environ["NCAR_HOST"] + del os.environ["NCAR_HOST"] else: self._ncarhost = None def tearDown(self): shutil.rmtree(self._tempdir, ignore_errors=True) if self._ncarhost is not None: - os.environ['NCAR_HOST'] = self._ncarhost + os.environ["NCAR_HOST"] = self._ncarhost def test_buildSetup_userDefinedMachine_minimalInfo(self): """Get through the case.setup phase with a user-defined machine diff --git a/python/ctsm/test/test_unit_args_utils.py b/python/ctsm/test/test_unit_args_utils.py index 3e7ec122bc..3a31b25224 100755 --- a/python/ctsm/test/test_unit_args_utils.py +++ b/python/ctsm/test/test_unit_args_utils.py @@ -12,9 +12,7 @@ import argparse # -- add python/ctsm to path (needed if we want to run the test stand-alone) -_CTSM_PYTHON = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir -) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) # pylint: disable=wrong-import-position @@ -50,9 +48,7 @@ def test_plonType_outOfBounds_positive(self): """ Test of plon values greater than 360 """ - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "Longitude.*should be between" - ): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "Longitude.*should be between"): _ = plon_type(360.5) # -- < -180 @@ -60,9 +56,7 @@ def test_plonType_outOfBounds_negative(self): """ Test of plon values smaller than -180 """ - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "Longitude.*should be between" - ): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "Longitude.*should be between"): _ = plon_type(-200) # -- = -180 @@ -99,9 +93,7 @@ def test_platType_outOfBounds_positive(self): """ Test of plat_type bigger than 90 """ - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "Latitude.*should be between" - ): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "Latitude.*should be between"): _ = plat_type(91) def test_platType_outOfBounds_pos90(self): @@ -122,9 +114,7 @@ def test_platType_outOfBounds_negative(self): """ Test of plat_type smaller than -90 """ - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "Latitude.*should be between" - ): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "Latitude.*should be between"): _ = plat_type(-91) diff --git a/python/ctsm/test/test_unit_lilac_build_ctsm.py b/python/ctsm/test/test_unit_lilac_build_ctsm.py index 12719697e2..1a3d020b52 100755 --- a/python/ctsm/test/test_unit_lilac_build_ctsm.py +++ b/python/ctsm/test/test_unit_lilac_build_ctsm.py @@ -50,9 +50,7 @@ def test_commandlineArgs_rebuild_invalid2(self, mock_stderr): """ expected_re = r"--os cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args( - args_to_parse=["build/directory", "--rebuild", "--os", "linux"] - ) + _ = _commandline_args(args_to_parse=["build/directory", "--rebuild", "--os", "linux"]) self.assertRegex(mock_stderr.getvalue(), expected_re) @patch("sys.stderr", new_callable=StringIO) @@ -82,9 +80,7 @@ def test_commandlineArgs_rebuild_invalid4(self, mock_stderr): """ expected_re = r"--no-build cannot be provided if --rebuild is set" with self.assertRaises(SystemExit): - _ = _commandline_args( - args_to_parse=["build/directory", "--rebuild", "--no-build"] - ) + _ = _commandline_args(args_to_parse=["build/directory", "--rebuild", "--no-build"]) self.assertRegex(mock_stderr.getvalue(), expected_re) @patch("sys.stderr", new_callable=StringIO) @@ -154,9 +150,7 @@ def test_commandlineArgs_noRebuild_invalid2(self, mock_stderr): This tests an argument in the new-machine-required list """ - expected_re = ( - r"--os must be provided if neither --machine nor --rebuild are set" - ) + expected_re = r"--os must be provided if neither --machine nor --rebuild are set" with self.assertRaises(SystemExit): _ = _commandline_args( args_to_parse=[ @@ -177,7 +171,9 @@ def test_commandlineArgs_noRebuild_invalid2(self, mock_stderr): @patch("sys.stderr", new_callable=StringIO) def test_commandlineArgs_noRebuild_invalidNeedToDictatePnetcdf(self, mock_stderr): """Test _commandline_args without --rebuild or --machine: invalid b/c nothing specified for pnetcdf""" - expected_re = r"For a user-defined machine, need to specify either --no-pnetcdf or --pnetcdf-path" + expected_re = ( + r"For a user-defined machine, need to specify either --no-pnetcdf or --pnetcdf-path" + ) with self.assertRaises(SystemExit): _ = _commandline_args( args_to_parse=[ @@ -242,9 +238,7 @@ def test_commandlineArgs_machine_missingRequired(self, mock_stderr): """Test _commandline_args with --machine, with a missing required argument""" expected_re = r"--compiler must be provided if --rebuild is not set" with self.assertRaises(SystemExit): - _ = _commandline_args( - args_to_parse=["build/directory", "--machine", "mymachine"] - ) + _ = _commandline_args(args_to_parse=["build/directory", "--machine", "mymachine"]) self.assertRegex(mock_stderr.getvalue(), expected_re) @patch("sys.stderr", new_callable=StringIO) diff --git a/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py b/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py index ec21204f8e..55e5028e8c 100755 --- a/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py +++ b/python/ctsm/test/test_unit_lilac_make_runtime_inputs.py @@ -23,9 +23,7 @@ def test_buildnmlOpts_bgc(self): def test_buildnmlOpts_fates(self): """Test determine_buildnml_opts with bgc_mode='fates'""" - bldnml_opts = determine_bldnml_opts( - bgc_mode="fates", crop="off", vichydro="off" - ) + bldnml_opts = determine_bldnml_opts(bgc_mode="fates", crop="off", vichydro="off") self.assertRegex(bldnml_opts, r"^ *-bgc fates +-no-megan *$") def test_buildnmlOpts_bgcCrop(self): diff --git a/python/ctsm/test/test_unit_machine.py b/python/ctsm/test/test_unit_machine.py index d7aa6aca3d..e57dabb346 100755 --- a/python/ctsm/test/test_unit_machine.py +++ b/python/ctsm/test/test_unit_machine.py @@ -60,9 +60,7 @@ def assertNoBatchInfo(self, machine, nice_level=None): nice_level = 0 self.assertEqual(launcher.get_nice_level(), nice_level) - def assertQsubInfo( - self, machine, queue, walltime, account, required_args, extra_args - ): + def assertQsubInfo(self, machine, queue, walltime, account, required_args, extra_args): """Asserts that the machine's launcher is of type JobLauncherQsub, and has values as expected""" launcher = machine.job_launcher @@ -99,9 +97,7 @@ def create_defaults(default_job_launcher=JOB_LAUNCHER_QSUB): def test_unknownMachine_defaults(self): """Tests a machine not in the defaults structure, with no overriding arguments""" - machine = create_machine( - "unknown_test_machine", MACHINE_DEFAULTS, account="a123" - ) + machine = create_machine("unknown_test_machine", MACHINE_DEFAULTS, account="a123") self.assertMachineInfo( machine=machine, name="unknown_test_machine", @@ -236,9 +232,7 @@ def test_baselineDir_default(self): def test_baselineDir_noDefault(self): """Tests get_possibly_overridden_mach_value when baseline_dir is not provided and there is no default""" - machine = create_machine( - "unknown_test_machine", MACHINE_DEFAULTS, account="a123" - ) + machine = create_machine("unknown_test_machine", MACHINE_DEFAULTS, account="a123") baseline_dir = get_possibly_overridden_mach_value( machine, varname="baseline_dir", value=None ) diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index dca8dfa99f..e239ac9916 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -137,9 +137,7 @@ def test_getNotRectangle_lon1leLon2Lat1leLat2(self): # Hardwire where I expect not_rectangle to be False (0) # I have chosen the lon/lat ranges to match their corresponding index # values to keep this simple - compare[ - lat_1 - min_lat : lat_2 - min_lat + 1, lon_1 - min_lon : lon_2 - min_lon + 1 - ] = 0 + compare[lat_1 - min_lat : lat_2 - min_lat + 1, lon_1 - min_lon : lon_2 - min_lon + 1] = 0 np.testing.assert_array_equal(not_rectangle, compare) def test_getNotRectangle_lon1leLon2Lat1gtLat2(self): diff --git a/python/ctsm/test/test_unit_run_sys_tests.py b/python/ctsm/test/test_unit_run_sys_tests.py index ba5b1b0ed1..ee5197d76f 100755 --- a/python/ctsm/test/test_unit_run_sys_tests.py +++ b/python/ctsm/test/test_unit_run_sys_tests.py @@ -78,9 +78,7 @@ def test_testroot_setup(self): machine = self._make_machine() with mock.patch("ctsm.run_sys_tests.datetime") as mock_date: mock_date.now.side_effect = self._fake_now - run_sys_tests( - machine=machine, cime_path=self._cime_path(), testlist=["foo"] - ) + run_sys_tests(machine=machine, cime_path=self._cime_path(), testlist=["foo"]) expected_dir = os.path.join(self._scratch, self._expected_testroot()) self.assertTrue(os.path.isdir(expected_dir)) @@ -114,16 +112,10 @@ def test_createTestCommand_testnames(self): self.assertEqual(len(all_commands), 1) command = all_commands[0].cmd expected_create_test = os.path.join(self._cime_path(), "scripts", "create_test") - six.assertRegex( - self, command, r"^ *{}\s".format(re.escape(expected_create_test)) - ) - six.assertRegex( - self, command, r"--test-id +{}\s".format(self._expected_testid()) - ) + six.assertRegex(self, command, r"^ *{}\s".format(re.escape(expected_create_test))) + six.assertRegex(self, command, r"--test-id +{}\s".format(self._expected_testid())) expected_testroot_path = os.path.join(self._scratch, self._expected_testroot()) - six.assertRegex( - self, command, r"--output-root +{}\s".format(expected_testroot_path) - ) + six.assertRegex(self, command, r"--output-root +{}\s".format(expected_testroot_path)) six.assertRegex(self, command, r"--retry +0(\s|$)") six.assertRegex(self, command, r"test1 +test2(\s|$)") assertNotRegex(self, command, r"--compare\s") @@ -171,9 +163,7 @@ def test_createTestCommand_testfileAndExtraArgs(self): command = all_commands[0].cmd six.assertRegex(self, command, r"--test-id +mytestid(\s|$)") expected_testroot = os.path.join(testroot_base, "tests_mytestid") - six.assertRegex( - self, command, r"--output-root +{}(\s|$)".format(expected_testroot) - ) + six.assertRegex(self, command, r"--output-root +{}(\s|$)".format(expected_testroot)) six.assertRegex(self, command, r"--testfile +/path/to/testfile(\s|$)") six.assertRegex(self, command, r"--compare +mycompare(\s|$)") six.assertRegex(self, command, r"--generate +mygenerate(\s|$)") @@ -207,16 +197,12 @@ def test_createTestCommands_testsuite(self): {"compiler": "pgi"}, {"compiler": "intel"}, ] - run_sys_tests( - machine=machine, cime_path=self._cime_path(), suite_name="my_suite" - ) + run_sys_tests(machine=machine, cime_path=self._cime_path(), suite_name="my_suite") all_commands = machine.job_launcher.get_commands() self.assertEqual(len(all_commands), 2) for command in all_commands: - six.assertRegex( - self, command.cmd, r"--xml-category +{}(\s|$)".format("my_suite") - ) + six.assertRegex(self, command.cmd, r"--xml-category +{}(\s|$)".format("my_suite")) six.assertRegex( self, command.cmd, r"--xml-machine +{}(\s|$)".format(self._MACHINE_NAME) ) @@ -226,12 +212,8 @@ def test_createTestCommands_testsuite(self): expected_testid1 = "{}_int".format(self._expected_testid()) expected_testid2 = "{}_pgi".format(self._expected_testid()) - six.assertRegex( - self, all_commands[0].cmd, r"--test-id +{}(\s|$)".format(expected_testid1) - ) - six.assertRegex( - self, all_commands[1].cmd, r"--test-id +{}(\s|$)".format(expected_testid2) - ) + six.assertRegex(self, all_commands[0].cmd, r"--test-id +{}(\s|$)".format(expected_testid1)) + six.assertRegex(self, all_commands[1].cmd, r"--test-id +{}(\s|$)".format(expected_testid2)) expected_testroot_path = os.path.join(self._scratch, self._expected_testroot()) self.assertEqual( @@ -251,9 +233,7 @@ def test_createTestCommands_testsuite(self): os.path.join(expected_testroot_path, "STDERR." + expected_testid2), ) - expected_cs_status = os.path.join( - self._scratch, self._expected_testroot(), "cs.status" - ) + expected_cs_status = os.path.join(self._scratch, self._expected_testroot(), "cs.status") expected_cs_status = os.path.join( self._scratch, self._expected_testroot(), "cs.status.fails" ) @@ -285,9 +265,7 @@ def test_createTestCommands_testsuiteSpecifiedCompilers(self): def test_withDryRun_nothingDone(self): """With dry_run=True, no directories should be created, and no commands should be run""" machine = self._make_machine() - run_sys_tests( - machine=machine, cime_path=self._cime_path(), testlist=["foo"], dry_run=True - ) + run_sys_tests(machine=machine, cime_path=self._cime_path(), testlist=["foo"], dry_run=True) self.assertEqual(os.listdir(self._scratch), []) self.assertEqual(machine.job_launcher.get_commands(), []) diff --git a/python/ctsm/test/test_unit_singlept_data.py b/python/ctsm/test/test_unit_singlept_data.py index b924d49762..570207fa26 100755 --- a/python/ctsm/test/test_unit_singlept_data.py +++ b/python/ctsm/test/test_unit_singlept_data.py @@ -12,9 +12,7 @@ import sys # -- add python/ctsm to path (needed if we want to run the test stand-alone) -_CTSM_PYTHON = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir -) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) # pylint: disable=wrong-import-position @@ -122,9 +120,7 @@ def test_check_dom_pft_too_big(self): overwrite=self.overwrite, ) single_point.dom_pft = [16, 36, 79] - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "values for --dompft should*" - ): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "values for --dompft should*"): single_point.check_dom_pft() def test_check_dom_pft_too_small(self): @@ -151,9 +147,7 @@ def test_check_dom_pft_too_small(self): overwrite=self.overwrite, ) single_point.dom_pft = [16, 36, -1] - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "values for --dompft should*" - ): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "values for --dompft should*"): single_point.check_dom_pft() def test_check_dom_pft_numpft(self): diff --git a/python/ctsm/test/test_unit_singlept_data_surfdata.py b/python/ctsm/test/test_unit_singlept_data_surfdata.py index 20614b2da3..9623975452 100755 --- a/python/ctsm/test/test_unit_singlept_data_surfdata.py +++ b/python/ctsm/test/test_unit_singlept_data_surfdata.py @@ -17,9 +17,7 @@ import xarray as xr # -- add python/ctsm to path (needed if we want to run the test stand-alone) -_CTSM_PYTHON = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir -) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) # pylint: disable=wrong-import-position @@ -29,6 +27,7 @@ # pylint: disable=invalid-name # pylint: disable=too-many-lines + class TestSinglePointCaseSurfaceNoCrop(unittest.TestCase): """ Basic class for testing creating and modifying surface dataset for @@ -229,9 +228,6 @@ def test_modify_surfdata_atpoint_nocrop_1pft_pctcrop(self): self.assertEqual(ds_out["PCT_CROP"].data[:, :], 0) - - - def test_modify_surfdata_atpoint_nocrop_1pft_glacier(self): """ Test modify_surfdata_atpoint @@ -516,7 +512,6 @@ def test_modify_surfdata_atpoint_nocrop_wetland_include_nonveg(self): self.assertNotEqual(ds_out["PCT_WETLAND"].data[:, :], 0) - def test_modify_surfdata_atpoint_nocrop_nopft_zero_nonveg(self): """ Test modify_surfdata_atpoint @@ -542,10 +537,10 @@ def test_modify_surfdata_atpoint_nocrop_nopft_zero_nonveg(self): ) single_point.dom_pft = None single_point.include_nonveg = False - self.ds_test['PCT_CROP'].values = [[40]] - self.ds_test['PCT_LAKE'].values = [[10]] - self.ds_test['PCT_WETLAND'].values = [[10]] - self.ds_test['PCT_NATVEG'].values = [[40]] + self.ds_test["PCT_CROP"].values = [[40]] + self.ds_test["PCT_LAKE"].values = [[10]] + self.ds_test["PCT_WETLAND"].values = [[10]] + self.ds_test["PCT_NATVEG"].values = [[40]] ds_out = single_point.modify_surfdata_atpoint(self.ds_test) self.assertEqual(ds_out["PCT_CROP"].data[:, :], 50) @@ -575,14 +570,15 @@ def test_modify_surfdata_atpoint_nocrop_nopft_include_nonveg(self): ) single_point.dom_pft = None single_point.include_nonveg = True - self.ds_test['PCT_CROP'].values = [[40]] - self.ds_test['PCT_LAKE'].values = [[10]] - self.ds_test['PCT_WETLAND'].values = [[10]] - self.ds_test['PCT_NATVEG'].values = [[40]] + self.ds_test["PCT_CROP"].values = [[40]] + self.ds_test["PCT_LAKE"].values = [[10]] + self.ds_test["PCT_WETLAND"].values = [[10]] + self.ds_test["PCT_NATVEG"].values = [[40]] ds_out = single_point.modify_surfdata_atpoint(self.ds_test) self.assertEqual(ds_out["PCT_CROP"].data[:, :], 40) + class TestSinglePointCaseSurfaceCrop(unittest.TestCase): """ Basic class for testing creating and modifying surface dataset for @@ -1088,10 +1084,10 @@ def test_modify_surfdata_atpoint_crop_nopft_zero_nonveg(self): ) single_point.dom_pft = None single_point.include_nonveg = False - self.ds_test['PCT_CROP'].values = [[40]] - self.ds_test['PCT_LAKE'].values = [[10]] - self.ds_test['PCT_WETLAND'].values = [[10]] - self.ds_test['PCT_NATVEG'].values = [[40]] + self.ds_test["PCT_CROP"].values = [[40]] + self.ds_test["PCT_LAKE"].values = [[10]] + self.ds_test["PCT_WETLAND"].values = [[10]] + self.ds_test["PCT_NATVEG"].values = [[40]] ds_out = single_point.modify_surfdata_atpoint(self.ds_test) self.assertEqual(ds_out["PCT_NATVEG"].data[:, :], 50) @@ -1121,14 +1117,15 @@ def test_modify_surfdata_atpoint_crop_nopft_include_nonveg(self): ) single_point.dom_pft = None single_point.include_nonveg = True - self.ds_test['PCT_CROP'].values = [[40]] - self.ds_test['PCT_LAKE'].values = [[10]] - self.ds_test['PCT_WETLAND'].values = [[10]] - self.ds_test['PCT_NATVEG'].values = [[40]] + self.ds_test["PCT_CROP"].values = [[40]] + self.ds_test["PCT_LAKE"].values = [[10]] + self.ds_test["PCT_WETLAND"].values = [[10]] + self.ds_test["PCT_NATVEG"].values = [[40]] ds_out = single_point.modify_surfdata_atpoint(self.ds_test) self.assertEqual(ds_out["PCT_NATVEG"].data[:, :], 40) + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index a4ad42a9e8..a78928abcc 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -93,9 +93,7 @@ def test_lonRange0To360_outOfBounds(self): """ Tests that lon_range_0_to_360 aborts gracefully when lon = 361 """ - with self.assertRaisesRegex( - SystemExit, "lon_in needs to be in the range 0 to 360" - ): + with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): _ = lon_range_0_to_360(361) diff --git a/python/ctsm/toolchain/ctsm_case.py b/python/ctsm/toolchain/ctsm_case.py index 2e2a5544f1..333af0833a 100755 --- a/python/ctsm/toolchain/ctsm_case.py +++ b/python/ctsm/toolchain/ctsm_case.py @@ -114,9 +114,7 @@ def __str__(self): return ( str(self.__class__) + "\n" - + "\n".join( - (str(key) + " = " + str(value) for key, value in self.__dict__.items()) - ) + + "\n".join((str(key) + " = " + str(value) for key, value in self.__dict__.items())) ) def check_endyear(self): @@ -152,9 +150,7 @@ def check_num_pft(self): self.num_pft = "78" else: self.num_pft = "16" - logger.debug( - " crop_flag = %s => num_pft = %i", self.crop_flag.__str__(), self.num_pft - ) + logger.debug(" crop_flag = %s => num_pft = %i", self.crop_flag.__str__(), self.num_pft) def build_landuse_filename(self): """ @@ -201,9 +197,7 @@ def create_landuse_file(self): self.decode_ssp_rcp() lu_input_fname = os.path.join( self.input_path, - "pftcftdynharv.0.25x0.25." - + self.ssp_rcp - + ".simyr2016-2100.c181217", + "pftcftdynharv.0.25x0.25." + self.ssp_rcp + ".simyr2016-2100.c181217", "mksrf_landuse_SSP" + self.ssp_val + "RCP" @@ -218,9 +212,7 @@ def create_landuse_file(self): # -- Check if the land-use input file exist: if not os.path.isfile(lu_input_fname): logger.debug("lu_input_fname: %s", lu_input_fname) - logger.warning( - "land-use input file does not exist for year: %i.", year - ) + logger.warning("land-use input file does not exist for year: %i.", year) # TODO: make the space/tab exactly the same as pl code: lu_line = lu_input_fname + "\t\t\t" + str(year) + "\n" @@ -286,10 +278,7 @@ def create_namelist_file(self): nl_template = ( "&clmexp\n" "nglcec = " + self.glc_nec + "\n" - "mksrf_fsoitex = " - + self.input_path - + "mksrf_soitex.10level.c201018.nc" - + "\n" + "mksrf_fsoitex = " + self.input_path + "mksrf_soitex.10level.c201018.nc" + "\n" "mksrf_forganic = " + self.input_path + "mksrf_organic_10level_5x5min_ISRIC-WISE-NCSCD_nlev7_c120830.nc" @@ -298,14 +287,8 @@ def create_namelist_file(self): + self.input_path + "mksrf_LakePnDepth_3x3min_simyr2004_csplk_c151015.nc" + "\n" - "mksrf_fwetlnd = " - + self.input_path - + "mksrf_lanwat.050425.nc" - + "\n" - "mksrf_fmax = " - + self.input_path - + "mksrf_fmax_3x3min_USGS_c120911.nc" - + "\n" + "mksrf_fwetlnd = " + self.input_path + "mksrf_lanwat.050425.nc" + "\n" + "mksrf_fmax = " + self.input_path + "mksrf_fmax_3x3min_USGS_c120911.nc" + "\n" "mksrf_fglacier = " + self.input_path + "mksrf_glacier_3x3min_simyr2000.c120926.nc" @@ -314,10 +297,7 @@ def create_namelist_file(self): + self.input_path + "mksrf_vocef_0.5x0.5_simyr2000.c110531.nc" + "\n" - "mksrf_furbtopo = " - + self.input_path - + "mksrf_topo.10min.c080912.nc" - + "\n" + "mksrf_furbtopo = " + self.input_path + "mksrf_topo.10min.c080912.nc" + "\n" "mksrf_fgdp = " + self.input_path + "mksrf_gdp_0.5x0.5_AVHRR_simyr2000.c130228.nc" diff --git a/python/six.py b/python/six.py index b5b008a2dc..b0d3bf4af9 100644 --- a/python/six.py +++ b/python/six.py @@ -234,9 +234,7 @@ class _MovedItems(_LazyModule): _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute( - "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" - ), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), @@ -244,9 +242,7 @@ class _MovedItems(_LazyModule): MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute( - "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" - ), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), @@ -255,9 +251,7 @@ class _MovedItems(_LazyModule): MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute( - "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" - ), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), @@ -271,9 +265,7 @@ class _MovedItems(_LazyModule): MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule( - "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" - ), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), @@ -343,9 +335,7 @@ class Module_six_moves_urllib_parse(_LazyModule): MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute( - "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes" - ), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), @@ -484,9 +474,7 @@ class Module_six_moves_urllib_robotparser(_LazyModule): setattr(Module_six_moves_urllib_robotparser, attr.name, attr) del attr -Module_six_moves_urllib_robotparser._moved_attributes = ( - _urllib_robotparser_moved_attributes -) +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes _importer._add_module( Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), @@ -510,9 +498,7 @@ def __dir__(self): return ["parse", "error", "request", "response", "robotparser"] -_importer._add_module( - Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" -) +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib") def add_move(move): @@ -595,9 +581,7 @@ def next(self): return type(self).__next__(self) callable = callable -_add_doc( - get_unbound_function, """Get the function out of a possibly unbound function""" -) +_add_doc(get_unbound_function, """Get the function out of a possibly unbound function""") get_method_function = operator.attrgetter(_meth_func) @@ -650,9 +634,7 @@ def iterlists(d, **kw): _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") _add_doc(itervalues, "Return an iterator over the values of a dictionary.") _add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc( - iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." -) +_add_doc(iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary.") if PY3: @@ -801,11 +783,7 @@ def write(data): if not isinstance(data, basestring): data = str(data) # If the file has an encoding, encode unicode with it. - if ( - isinstance(fp, file) - and isinstance(data, unicode) - and fp.encoding is not None - ): + if isinstance(fp, file) and isinstance(data, unicode) and fp.encoding is not None: errors = getattr(fp, "errors", None) if errors is None: errors = "strict" @@ -949,10 +927,7 @@ def python_2_unicode_compatible(klass): # be floating around. Therefore, we can't use isinstance() to check for # the six meta path importer, since the other six instance will have # inserted an importer with different class. - if ( - type(importer).__name__ == "_SixMetaPathImporter" - and importer.name == __name__ - ): + if type(importer).__name__ == "_SixMetaPathImporter" and importer.name == __name__: del sys.meta_path[i] break del i, importer From 67bb51edb951474f387cebd30015aed705a9d506 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 6 Jun 2022 12:30:57 -0600 Subject: [PATCH 09/16] Add black format to git-blame --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 62cd314d46..4df8d2b62b 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,4 @@ # Ran python directory through black python formatter +4cd83cb3ee6d85eb909403487abf5eeaf4d98911 0aa2957c1f8603c63fa30b11295c06cfddff44a5 2cdb380febb274478e84cd90945aee93f29fa2e6 From 3c8b6968879a52179a2d880018a53d3842d8a93e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 01:12:10 -0600 Subject: [PATCH 10/16] Go away from the jpetrucciani version as it was not working, try the psf version as recommended by black in the 22.3.0 documentation on github actions --- .github/workflows/black.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index d351f04817..9f3fc04911 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -10,10 +10,10 @@ jobs: steps: # Checkout the code - uses: actions/checkout@v2 - # Use a specific version of the black-checker - # Use version that's the same as with conda on cheyenne - # Tag for black-check corresponds to the black version tag - - uses: jpetrucciani/black-check@21.12b0 + # Use the latest stable version of the github action + - uses: psf/black@stable with: - path: python - black_flags: --config python/pyproject.toml + # Use options and version identical to the conda environment + options: "--check --config python/pyproject.toml" + src: "./python" + version: "22.3.0" From b03efc12702e8b3d943957623f9179f52c383720 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 13:09:58 -0600 Subject: [PATCH 11/16] Apply for all python files, just to see the check fail --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 9f3fc04911..8346508257 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -15,5 +15,5 @@ jobs: with: # Use options and version identical to the conda environment options: "--check --config python/pyproject.toml" - src: "./python" + src: "./" version: "22.3.0" From be03e4f0866dd503792fd12e3939435224424a6a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 13:13:38 -0600 Subject: [PATCH 12/16] Add diff option as directed to in black github actions example --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 8346508257..97de6749ee 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -14,6 +14,6 @@ jobs: - uses: psf/black@stable with: # Use options and version identical to the conda environment - options: "--check --config python/pyproject.toml" + options: "--check --diff --config python/pyproject.toml" src: "./" version: "22.3.0" From bfc444311dc17014f1896b09a1e0588ea96de978 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 13:40:35 -0600 Subject: [PATCH 13/16] Rename so matches the filename --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 97de6749ee..55ca9f28a3 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,4 @@ -name: push black workflow +name: black # # Run the python formatting in check mode # From ca16175357d58a473e8162c043d2e0c3027ef79f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 13:43:46 -0600 Subject: [PATCH 14/16] Apply only to the python directory --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 55ca9f28a3..29aba40788 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -15,5 +15,5 @@ jobs: with: # Use options and version identical to the conda environment options: "--check --diff --config python/pyproject.toml" - src: "./" + src: "./python" version: "22.3.0" From a8ba1c1639075350891f9a1a449a3fc84ea83e34 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 14:13:18 -0600 Subject: [PATCH 15/16] Change the action to add PR to trigger add more description, remove the --diff option so just lists files that would change --- .github/workflows/black.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 29aba40788..5fcf7fd7f0 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,8 +1,8 @@ -name: black +name: black check on push and PR # # Run the python formatting in check mode # -on: push +on: [push, pull_request] jobs: black-check: @@ -14,6 +14,6 @@ jobs: - uses: psf/black@stable with: # Use options and version identical to the conda environment - options: "--check --diff --config python/pyproject.toml" + options: "--check --config python/pyproject.toml" src: "./python" version: "22.3.0" From 012a2f5ac81cd3fc8fc328521e0cba98398023d4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Jun 2022 14:20:23 -0600 Subject: [PATCH 16/16] Test for all --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 5fcf7fd7f0..b792ea240e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -15,5 +15,5 @@ jobs: with: # Use options and version identical to the conda environment options: "--check --config python/pyproject.toml" - src: "./python" + src: "./" version: "22.3.0"