From 562cf4b622e935d9be13782f9eac8583dd223ba5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 13 Apr 2017 13:50:56 -0600 Subject: [PATCH 01/21] add support for user_mods in compset definition --- config/xml_schemas/config_compsets.xsd | 2 ++ scripts/lib/CIME/XML/compsets.py | 4 +++- scripts/lib/CIME/case.py | 30 +++++++++++++++++--------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/config/xml_schemas/config_compsets.xsd b/config/xml_schemas/config_compsets.xsd index e9072738a6d..e2b64df2c66 100644 --- a/config/xml_schemas/config_compsets.xsd +++ b/config/xml_schemas/config_compsets.xsd @@ -11,6 +11,7 @@ + @@ -32,6 +33,7 @@ + diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 875aea5d99a..db1c4c62d78 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -36,8 +36,10 @@ def get_compset_match(self, name): for node in science_support_nodes: science_support.append(node.get("grid")) + user_mods = self.get_optional_node("user_mods", root=node) + logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) - return (lname, alias, science_support) + return (lname, alias, science_support, user_mods) return (None, None, [False]) def get_compset_var_settings(self, compset, grid): diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index ca727570fa7..5a81b0fba12 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -95,7 +95,9 @@ def __init__(self, case_root=None, read_only=True): self._components = [] self._component_classes = [] self._is_env_loaded = False - + # these are user_mods as defined in the compset + # Command Line user_mods are handeled seperately + self._user_mods = None self.thread_count = None self.total_tasks = None self.tasks_per_node = None @@ -419,7 +421,7 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf # If the file exists, read it and see if there is a match for the compset alias or longname if (os.path.isfile(compsets_filename)): compsets = Compsets(compsets_filename) - match, compset_alias, science_support = compsets.get_compset_match(name=compset_name) + match, compset_alias, science_support, self._user_mods = compsets.get_compset_match(name=compset_name) pesfile = files.get_value("PES_SPEC_FILE" , {"component":component}) if match is not None: self._pesfile = pesfile @@ -952,14 +954,22 @@ def create_caseroot(self, clone=False): self._create_caseroot_tools() def apply_user_mods(self, user_mods_dir=None): - if user_mods_dir is not None: - if os.path.isabs(user_mods_dir): - user_mods_path = user_mods_dir - else: - user_mods_path = self.get_value('USER_MODS_DIR') - user_mods_path = os.path.join(user_mods_path, user_mods_dir) - self.set_value("USER_MODS_FULLPATH",user_mods_path) - apply_user_mods(self._caseroot, user_mods_path) + """ + User mods can be specified on the create_newcase command line (usually when called from create test) + or they can be in the compset definition, or both. + """ + + for user_mods in (user_mods_dir, self._user_mods): + if user_mods is not None: + if os.path.isabs(user_mods): + user_mods_path = user_mods + else: + user_mods_path = self.get_value('USER_MODS_DIR') + user_mods_path = os.path.join(user_mods_path, user_mods) + CIME.user_mod_support.apply_user_mods(self._caseroot, user_mods_path) + + + def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime_output_root=None): if cime_output_root is None: From efb71259f4a6557ebdf56fd6a789e5c11226d888 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 13 Apr 2017 14:18:17 -0600 Subject: [PATCH 02/21] fix pylint issue --- scripts/lib/CIME/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 5a81b0fba12..f2aa9825ca3 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -966,7 +966,7 @@ def apply_user_mods(self, user_mods_dir=None): else: user_mods_path = self.get_value('USER_MODS_DIR') user_mods_path = os.path.join(user_mods_path, user_mods) - CIME.user_mod_support.apply_user_mods(self._caseroot, user_mods_path) + apply_user_mods(self._caseroot, user_mods_path) From ea4a759ee3a44b8e99c132549e784b0d1569806b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 13 Apr 2017 14:30:15 -0600 Subject: [PATCH 03/21] fix no-match args --- scripts/lib/CIME/XML/compsets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index db1c4c62d78..09048b0e148 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -27,7 +27,9 @@ def get_compset_match(self, name): nodes = self.get_nodes("compset") alias = None lname = None + science_support = [] + for node in nodes: alias = self.get_element_text("alias",root=node) lname = self.get_element_text("lname",root=node) @@ -40,7 +42,7 @@ def get_compset_match(self, name): logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support, user_mods) - return (None, None, [False]) + return (None, None, [False], None) def get_compset_var_settings(self, compset, grid): ''' From 685c3febf3a3e6fff6984149fb6a8862020698c5 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Fri, 14 Apr 2017 14:14:12 -0600 Subject: [PATCH 04/21] first changes needed for aquaplanet som --- config/cesm/config_grids.xml | 25 ++++++++++++++++--- config/xml_schemas/config_grids_v2.xsd | 2 +- .../docn/cime_config/config_component.xml | 7 ++++-- .../cime_config/namelist_definition_docn.xml | 4 +++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/config/cesm/config_grids.xml b/config/cesm/config_grids.xml index 269839e58c7..8fab255610d 100644 --- a/config/cesm/config_grids.xml +++ b/config/cesm/config_grids.xml @@ -361,13 +361,20 @@ gx1v7 - + 0.9x1.25 0.9x1.25 0.9x1.25 gx1v6 + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + null + + 0.9x1.25 0.9x1.25 @@ -506,6 +513,13 @@ gx1v6 + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + null + + 1.9x2.5 1.9x2.5 @@ -959,9 +973,11 @@ 288 192 domain.lnd.fv0.9x1.25_gx1v6.090309.nc - domain.ocn.0.9x1.25_gx1v6_090403.nc + domain.ocn.0.9x1.25_gx1v6_090403.nc domain.lnd.fv0.9x1.25_gx1v7.151020.nc - domain.ocn.fv0.9x1.25_gx1v7.151020.nc + domain.ocn.fv0.9x1.25_gx1v7.151020.nc + /glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc + /glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc 0.9x1.25 is FV 1-deg grid: @@ -969,6 +985,7 @@ 144 96 domain.lnd.fv1.9x2.5_gx1v6.090206.nc domain.ocn.1.9x2.5_gx1v6_090403.nc + domain.aqua.fv1.9x2.5.nc 1.9x2.5 is FV 2-deg grid: @@ -1483,7 +1500,7 @@ cpl/gridmaps/gx1v7/map_gx1v7_TO_ww3a_splice_170214.nc - + cpl/gridmaps/T31/map_T31_TO_ww3a_bilin_131104.nc diff --git a/config/xml_schemas/config_grids_v2.xsd b/config/xml_schemas/config_grids_v2.xsd index 2bac0a1de03..b4680b93f9e 100644 --- a/config/xml_schemas/config_grids_v2.xsd +++ b/config/xml_schemas/config_grids_v2.xsd @@ -4,7 +4,7 @@ - + diff --git a/src/components/data_comps/docn/cime_config/config_component.xml b/src/components/data_comps/docn/cime_config/config_component.xml index 165d21c1dad..686ab1bea33 100644 --- a/src/components/data_comps/docn/cime_config/config_component.xml +++ b/src/components/data_comps/docn/cime_config/config_component.xml @@ -15,7 +15,7 @@ char - prescribed,som,copyall,interannual,null + prescribed,pres_aquap,som,som_aquap,copyall,interannual,null prescribed null @@ -24,6 +24,8 @@ us20 interannual copyall + pres_aquap + som_aquap run_component_docn env_run.xml @@ -69,7 +71,8 @@ UNSET - pop_frc.1x1d.090130.nc + pop_frc.1x1d.090130.nc + /glade/u/home/benedict/ys/datain/cesm2_0_beta03.som.forcing/cam4.som.forcing.aquaplanet.QzaFix_h30Fix_TspunFix.fv19_CTL.nc run_component_docn env_run.xml diff --git a/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 32d2d7f5ba5..5d249cdbd80 100644 --- a/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -47,7 +47,9 @@ List of streams used for the given docn_mode. prescribed + prescribed som + som interannual copyall @@ -312,7 +314,9 @@ NULL SSTDATA + SSTDATA SOM + SOM IAF COPYALL From 9cc7740ca19723a3a7b056f0e63e7abc28450132 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 14 Apr 2017 20:34:34 -0600 Subject: [PATCH 05/21] Fix precedence of user_mods application Now --user-mods on the command line (including testmods) will take precedence over the user_mods set by the compset - for user_nl files, shell_commands and SourceMods. I have tested this with this diff to the A compset diff --git a/src/drivers/mct/cime_config/config_compsets.xml b/src/drivers/mct/cime_config/config_compsets.xml index c11354e..7e6c2c9 100644 --- a/src/drivers/mct/cime_config/config_compsets.xml +++ b/src/drivers/mct/cime_config/config_compsets.xml @@ -40,6 +40,7 @@ A 2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV + /Users/sacks/temporary/user_mods_compset Along with this create_newcase command: ./create_newcase -case test_0414m -compset A -res f45_g37 \ --run-unsupported \ --user-mods-dir /Users/sacks/temporary/user_mods_command_line where the contents of the two relevant user_mods directories are: --- user_mods_compset/shell_commands --- ./xmlchange STOP_N=101 --- user_mods_compset/SourceMods/src.drv/mysrc.F90 --- user_mods_compset --- user_mods_compset/user_nl_cpl --- user_mods_compset --- user_mods_command_line/shell_commands --- ./xmlchange STOP_N=102 --- user_mods_command_line/SourceMods/src.drv/mysrc.F90 --- user_mods_command_line --- user_mods_command_line/user_nl_cpl --- user_mods_command_line The final contents are: --- user_nl_cpl --- user_mods_compset user_mods_command_line --- shell_commands --- ./xmlchange --force STOP_N=102 --- SourceMods/src.drv/mysrc.F90 --- user_mods_command_line And $ ./xmlquery STOP_N STOP_N: 102 thus demonstrating that the user_mods on the command-line takes precedence over the compset's user_mods. --- scripts/lib/CIME/XML/compsets.py | 6 +++- scripts/lib/CIME/case.py | 6 ++-- scripts/lib/CIME/user_mod_support.py | 47 +++++++++++++++------------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 09048b0e148..f5b85d8a5d2 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -38,7 +38,11 @@ def get_compset_match(self, name): for node in science_support_nodes: science_support.append(node.get("grid")) - user_mods = self.get_optional_node("user_mods", root=node) + user_mods_node = self.get_optional_node("user_mods", root=node) + if user_mods_node is not None: + user_mods = user_mods_node.text + else: + user_mods = None logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support, user_mods) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index f2aa9825ca3..542be4948f3 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -958,8 +958,10 @@ def apply_user_mods(self, user_mods_dir=None): User mods can be specified on the create_newcase command line (usually when called from create test) or they can be in the compset definition, or both. """ - - for user_mods in (user_mods_dir, self._user_mods): + + # This looping order will lead to the specified user_mods_dir taking + # precedence over self._user_mods, if there are any conflicts. + for user_mods in (self._user_mods, user_mods_dir): if user_mods is not None: if os.path.isabs(user_mods): user_mods_path = user_mods diff --git a/scripts/lib/CIME/user_mod_support.py b/scripts/lib/CIME/user_mod_support.py index 572910e43b2..d27c7255542 100644 --- a/scripts/lib/CIME/user_mod_support.py +++ b/scripts/lib/CIME/user_mod_support.py @@ -14,6 +14,9 @@ def apply_user_mods(caseroot, user_mods_path): updating SourceMods and creating case shell_commands and xmlchange_cmds files First remove case shell_commands files if any already exist + + If this function is called multiple times, settings from later calls will + take precedence over earlier calls, if there are conflicts. ''' case_shell_command_files = [os.path.join(caseroot,"shell_commands"), os.path.join(caseroot,"xmlchange_cmnds")] @@ -22,6 +25,13 @@ def apply_user_mods(caseroot, user_mods_path): os.remove(shell_command_file) include_dirs = build_include_dirs_list(user_mods_path) + # If a user_mods dir 'foo' includes 'bar', the include_dirs list returned + # from build_include_dirs has 'foo' before 'bar'. But with the below code, + # directories that occur later in the list take precedence over the earlier + # ones, and we want 'foo' to take precedence over 'bar' in this case (in + # general: we want a given user_mods directory to take precedence over any + # mods that it includes). So we reverse include_dirs to accomplish this. + include_dirs.reverse() logger.debug("include_dirs are %s"%include_dirs) for include_dir in include_dirs: # write user_nl_xxx file in caseroot @@ -31,7 +41,11 @@ def apply_user_mods(caseroot, user_mods_path): if len(newcontents) == 0: continue case_user_nl = user_nl.replace(include_dir, caseroot) - update_user_nl_file(case_user_nl, newcontents) + # If the same variable is set twice in a user_nl file, the later one + # takes precedence. So by appending the new contents, later entries + # in the include_dirs list take precedence over earlier entries. + with open(case_user_nl, "a") as fd: + fd.write(newcontents) # update SourceMods in caseroot for root, _, files in os.walk(include_dir,followlinks=True,topdown=False): @@ -39,21 +53,18 @@ def apply_user_mods(caseroot, user_mods_path): for sfile in files: source_mods = os.path.join(root,sfile) case_source_mods = source_mods.replace(include_dir, caseroot) + # We overwrite any existing SourceMods file so that later + # include_dirs take precedence over earlier ones if os.path.isfile(case_source_mods): - logger.warn("Refusing to overwrite existing SourceMods in %s"%case_source_mods) + logger.warn("WARNING: Overwriting existing SourceMods in %s"%case_source_mods) else: logger.info("Adding SourceMod to case %s"%case_source_mods) - try: - shutil.copyfile(source_mods, case_source_mods) - except: - expect(False, "Could not write file %s in caseroot %s" - %(case_source_mods,caseroot)) + try: + shutil.copyfile(source_mods, case_source_mods) + except: + expect(False, "Could not write file %s in caseroot %s" + %(case_source_mods,caseroot)) - # Reverse include_dirs to make sure xmlchange commands are called in the - # correct order; it may be desireable to reverse include_dirs above the - # previous loop and then append user_nl changes rather than prepend them. - include_dirs.reverse() - for include_dir in include_dirs: # create xmlchange_cmnds and shell_commands in caseroot shell_command_files = glob.glob(os.path.join(include_dir,"shell_commands")) +\ glob.glob(os.path.join(include_dir,"xmlchange_cmnds")) @@ -70,6 +81,8 @@ def apply_user_mods(caseroot, user_mods_path): shell_commands_file) with open(shell_commands_file,"r") as fd: new_shell_commands = fd.read().replace("xmlchange","xmlchange --force") + # By appending the new commands to the end, settings from later + # include_dirs take precedence over earlier ones with open(case_shell_commands, "a") as fd: fd.write(new_shell_commands) @@ -78,16 +91,6 @@ def apply_user_mods(caseroot, user_mods_path): os.chmod(shell_command_file, 0777) run_cmd_no_fail(shell_command_file) -def update_user_nl_file(case_user_nl, contents): - if os.path.isfile(case_user_nl): - with open(case_user_nl, "r") as fd: - old_contents = fd.read() - contents = contents + old_contents - logger.debug("Pre-pending file %s"%(case_user_nl)) - with open(case_user_nl, "w") as fd: - fd.write(contents) - - def build_include_dirs_list(user_mods_path, include_dirs=None): ''' From 866493330903ac1311cefc62752a73f6911f7fc9 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Sat, 15 Apr 2017 11:04:56 -0600 Subject: [PATCH 06/21] fixed issue with using new user_mods element in compset definition --- scripts/lib/CIME/XML/compsets.py | 2 -- scripts/lib/CIME/case.py | 9 +++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index f5b85d8a5d2..2d96aaed3e6 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -37,13 +37,11 @@ def get_compset_match(self, name): science_support_nodes = self.get_nodes("science_support", root=node) for node in science_support_nodes: science_support.append(node.get("grid")) - user_mods_node = self.get_optional_node("user_mods", root=node) if user_mods_node is not None: user_mods = user_mods_node.text else: user_mods = None - logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support, user_mods) return (None, None, [False], None) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 542be4948f3..ae878745eb9 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -953,12 +953,16 @@ def create_caseroot(self, clone=False): self._create_caseroot_sourcemods() self._create_caseroot_tools() + # Apply user_mods if part of compset + if self._user_mods is not None: + self._user_mods = self.get_resolved_value(self._user_mods) + self.apply_user_mods() + def apply_user_mods(self, user_mods_dir=None): """ User mods can be specified on the create_newcase command line (usually when called from create test) or they can be in the compset definition, or both. """ - # This looping order will lead to the specified user_mods_dir taking # precedence over self._user_mods, if there are any conflicts. for user_mods in (self._user_mods, user_mods_dir): @@ -970,9 +974,6 @@ def apply_user_mods(self, user_mods_dir=None): user_mods_path = os.path.join(user_mods_path, user_mods) apply_user_mods(self._caseroot, user_mods_path) - - - def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime_output_root=None): if cime_output_root is None: cime_output_root = self.get_value("CIME_OUTPUT_ROOT") From 01ad74425c514da65ac6e6f741dd215de27166ac Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 16 Apr 2017 06:12:03 -0600 Subject: [PATCH 07/21] Add unit tests for user_mod_support --- .../lib/CIME/tests/test_user_mod_support.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 scripts/lib/CIME/tests/test_user_mod_support.py diff --git a/scripts/lib/CIME/tests/test_user_mod_support.py b/scripts/lib/CIME/tests/test_user_mod_support.py new file mode 100644 index 00000000000..c8e582f01b1 --- /dev/null +++ b/scripts/lib/CIME/tests/test_user_mod_support.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +import unittest +import shutil +import tempfile +import os +from CIME.user_mod_support import apply_user_mods + +# ======================================================================== +# Define some parameters +# ======================================================================== + +_SOURCEMODS = os.path.join("SourceMods", "src.drv") + +class TestUserModSupport(unittest.TestCase): + + # ======================================================================== + # Test helper functions + # ======================================================================== + + def setUp(self): + self._caseroot = tempfile.mkdtemp() + self._caseroot_sourcemods = os.path.join(self._caseroot, _SOURCEMODS) + os.makedirs(self._caseroot_sourcemods) + self._user_mods_parent_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._caseroot, ignore_errors=True) + shutil.rmtree(self._user_mods_parent_dir, ignore_errors=True) + + def createUserMod(self, name, include_dirs=None): + """Create a user_mods directory with the given name. + + This directory is created within self._user_mods_parent_dir + + For name='foo', it will contain: + + - A user_nl_cpl file with contents: + foo + + - A shell_commands file with contents: + echo foo >> /PATH/TO/CASEROOT/shell_commands_result + + - A file in _SOURCEMODS named myfile.F90 with contents: + foo + + If include_dirs is given, it should be a list of strings, giving names + of other user_mods directories to include. e.g., if include_dirs is + ['foo1', 'foo2'], then this will create a file 'include_user_mods' that + contains paths to the 'foo1' and 'foo2' user_mods directories, one per + line. + """ + + mod_dir = os.path.join(self._user_mods_parent_dir, name) + os.makedirs(mod_dir) + mod_dir_sourcemods = os.path.join(mod_dir, _SOURCEMODS) + os.makedirs(mod_dir_sourcemods) + + with open(os.path.join(mod_dir, "user_nl_cpl"), "w") as user_nl_cpl: + user_nl_cpl.write(name + "\n") + with open(os.path.join(mod_dir, "shell_commands"), "w") as shell_commands: + command = "echo %s >> %s/shell_commands_result\n"%(name, + self._caseroot) + shell_commands.write(command) + with open(os.path.join(mod_dir_sourcemods, "myfile.F90"), "w") as f90_file: + f90_file.write(name + "\n") + + if include_dirs: + with open(os.path.join(mod_dir, "include_user_mods"), "w") as include_user_mods: + for one_include in include_dirs: + include_user_mods.write(os.path.join(self._user_mods_parent_dir, one_include) + "\n") + + def assertResults(self, expected_user_nl_cpl, + expected_shell_commands_result, + expected_sourcemod, + msg = ""): + """Asserts that the contents of the files in self._caseroot match expectations + + If msg is provided, it is printed for some failing assertions + """ + + path_to_user_nl_cpl = os.path.join(self._caseroot, "user_nl_cpl") + self.assertTrue(os.path.isfile(path_to_user_nl_cpl), + msg = msg + ": user_nl_cpl does not exist") + with open(path_to_user_nl_cpl, "r") as user_nl_cpl: + contents = user_nl_cpl.read() + self.assertEqual(expected_user_nl_cpl, contents) + + path_to_shell_commands_result = os.path.join(self._caseroot, "shell_commands_result") + self.assertTrue(os.path.isfile(path_to_shell_commands_result), + msg = msg + ": shell_commands_result does not exist") + with open(path_to_shell_commands_result, "r") as shell_commands_result: + contents = shell_commands_result.read() + self.assertEqual(expected_shell_commands_result, contents) + + path_to_sourcemod = os.path.join(self._caseroot_sourcemods, "myfile.F90") + self.assertTrue(os.path.isfile(path_to_sourcemod), + msg = msg + ": sourcemod file does not exist") + with open(path_to_sourcemod, "r") as sourcemod: + contents = sourcemod.read() + self.assertEqual(expected_sourcemod, contents) + + # ======================================================================== + # Begin actual tests + # ======================================================================== + + def test_basic(self): + self.createUserMod("foo") + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo")) + self.assertResults(expected_user_nl_cpl = "foo\n", + expected_shell_commands_result = "foo\n", + expected_sourcemod = "foo\n", + msg = "test_basic") + + def test_two_applications(self): + """If apply_user_mods is called twice, the second should appear after the first so that it takes precedence.""" + + self.createUserMod("foo1") + self.createUserMod("foo2") + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo1")) + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo2")) + self.assertResults(expected_user_nl_cpl = "foo1\nfoo2\n", + expected_shell_commands_result = "foo1\nfoo2\n", + expected_sourcemod = "foo2\n", + msg = "test_two_applications") + + def test_include(self): + """If there is an included mod, the main one should appear after the included one so that it takes precedence.""" + + self.createUserMod("base") + self.createUserMod("derived", include_dirs=["base"]) + + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "derived")) + + self.assertResults(expected_user_nl_cpl = "base\nderived\n", + expected_shell_commands_result = "base\nderived\n", + expected_sourcemod = "derived\n", + msg = "test_include") + + def test_duplicate_includes(self): + """Test multiple includes, where both include the same base mod. + + The base mod should only be included once. + """ + + self.createUserMod("base") + self.createUserMod("derived1", include_dirs=["base"]) + self.createUserMod("derived2", include_dirs=["base"]) + self.createUserMod("derived_combo", + include_dirs = ["derived1", "derived2"]) + + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "derived_combo")) + + # NOTE(wjs, 2017-04-15) The ordering of derived1 vs. derived2 is not + # critical here: If this aspect of the behavior changes, the + # expected_contents can be changed to match the new behavior in this + # respect. + expected_contents = """base +derived2 +derived1 +derived_combo +""" + self.assertResults(expected_user_nl_cpl = expected_contents, + expected_shell_commands_result = expected_contents, + expected_sourcemod = "derived_combo\n", + msg = "test_duplicate_includes") From 8e8fd62a6b642e249d89cbf853827a850c045a85 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 16 Apr 2017 20:40:27 -0600 Subject: [PATCH 08/21] fix typo --- scripts/lib/CIME/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index ae878745eb9..f67a34ca095 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -96,7 +96,7 @@ def __init__(self, case_root=None, read_only=True): self._component_classes = [] self._is_env_loaded = False # these are user_mods as defined in the compset - # Command Line user_mods are handeled seperately + # Command Line user_mods are handled seperately self._user_mods = None self.thread_count = None self.total_tasks = None From ec95467d75c45047130104e6ca52acdcdc8546ad Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 16 Apr 2017 21:20:55 -0600 Subject: [PATCH 09/21] Fix timing of applying compset user_mods The previous implementation had two problems: 1. If you specified a user_mods on the command-line along with a compset that has its own user_mods, then the compset's user_mods get applied twice. 2. The new place where there was a call to apply_user_mods happened too early: xmlchange commands can not be done at that point. This fixes these problems. I have tested this with the same changes described in 9cc7740ca19723a3a7b056f0e63e7abc28450132. I tested create_newcase with no user_mods, user_mods just from the command line, user_mods just from the compset, and user_mods from the command line and the compset. --- scripts/create_newcase | 2 +- scripts/lib/CIME/case.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/create_newcase b/scripts/create_newcase index f1877cd76e5..a28f8c8d129 100755 --- a/scripts/create_newcase +++ b/scripts/create_newcase @@ -195,7 +195,7 @@ def _main_func(description): if user_mods_dir is not None: if os.path.isdir(user_mods_dir): user_mods_dir = os.path.abspath(user_mods_dir) - case.apply_user_mods(user_mods_dir) + case.apply_user_mods(user_mods_dir) # Lock env_case.xml lock_file("env_case.xml", caseroot) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index f67a34ca095..ec77ebb4ee5 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -953,19 +953,20 @@ def create_caseroot(self, clone=False): self._create_caseroot_sourcemods() self._create_caseroot_tools() - # Apply user_mods if part of compset - if self._user_mods is not None: - self._user_mods = self.get_resolved_value(self._user_mods) - self.apply_user_mods() - def apply_user_mods(self, user_mods_dir=None): """ User mods can be specified on the create_newcase command line (usually when called from create test) or they can be in the compset definition, or both. """ + + if self._user_mods is None: + compset_user_mods_resolved = None + else: + compset_user_mods_resolved = self.get_resolved_value(self._user_mods) + # This looping order will lead to the specified user_mods_dir taking # precedence over self._user_mods, if there are any conflicts. - for user_mods in (self._user_mods, user_mods_dir): + for user_mods in (compset_user_mods_resolved, user_mods_dir): if user_mods is not None: if os.path.isabs(user_mods): user_mods_path = user_mods From 8075121fcf298cc762e42667a21a855b1a29c7fb Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Apr 2017 09:43:01 -0600 Subject: [PATCH 10/21] add comment to README and stdout --- scripts/lib/CIME/case.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index ec77ebb4ee5..597876a4e8a 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -949,6 +949,10 @@ def create_caseroot(self, clone=False): comp_grid = "%s_GRID"%component_class append_status("%s is %s"%(comp_grid,self.get_value(comp_grid)), "README.case", caseroot=self._caseroot) + if self._user_mods is not None: + note = "This compset includes user_mods %s"%self._user_mods + append_status(note, "README.case", caseroot=self._caseroot) + logger.info(note) if not clone: self._create_caseroot_sourcemods() self._create_caseroot_tools() From 29e0159f7fba14e29141c184d5f8224517ecfb58 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 13 Apr 2017 13:50:56 -0600 Subject: [PATCH 11/21] add support for user_mods in compset definition --- config/xml_schemas/config_compsets.xsd | 2 ++ scripts/lib/CIME/XML/compsets.py | 4 +++- scripts/lib/CIME/case.py | 30 +++++++++++++++++--------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/config/xml_schemas/config_compsets.xsd b/config/xml_schemas/config_compsets.xsd index e9072738a6d..e2b64df2c66 100644 --- a/config/xml_schemas/config_compsets.xsd +++ b/config/xml_schemas/config_compsets.xsd @@ -11,6 +11,7 @@ + @@ -32,6 +33,7 @@ + diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 875aea5d99a..db1c4c62d78 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -36,8 +36,10 @@ def get_compset_match(self, name): for node in science_support_nodes: science_support.append(node.get("grid")) + user_mods = self.get_optional_node("user_mods", root=node) + logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) - return (lname, alias, science_support) + return (lname, alias, science_support, user_mods) return (None, None, [False]) def get_compset_var_settings(self, compset, grid): diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index e6e8531a27b..6add22645dc 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -95,7 +95,9 @@ def __init__(self, case_root=None, read_only=True): self._components = [] self._component_classes = [] self._is_env_loaded = False - + # these are user_mods as defined in the compset + # Command Line user_mods are handeled seperately + self._user_mods = None self.thread_count = None self.total_tasks = None self.tasks_per_node = None @@ -419,7 +421,7 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf # If the file exists, read it and see if there is a match for the compset alias or longname if (os.path.isfile(compsets_filename)): compsets = Compsets(compsets_filename) - match, compset_alias, science_support = compsets.get_compset_match(name=compset_name) + match, compset_alias, science_support, self._user_mods = compsets.get_compset_match(name=compset_name) pesfile = files.get_value("PES_SPEC_FILE" , {"component":component}) if match is not None: self._pesfile = pesfile @@ -963,14 +965,22 @@ def create_caseroot(self, clone=False): self._create_caseroot_tools() def apply_user_mods(self, user_mods_dir=None): - if user_mods_dir is not None: - if os.path.isabs(user_mods_dir): - user_mods_path = user_mods_dir - else: - user_mods_path = self.get_value('USER_MODS_DIR') - user_mods_path = os.path.join(user_mods_path, user_mods_dir) - self.set_value("USER_MODS_FULLPATH",user_mods_path) - apply_user_mods(self._caseroot, user_mods_path) + """ + User mods can be specified on the create_newcase command line (usually when called from create test) + or they can be in the compset definition, or both. + """ + + for user_mods in (user_mods_dir, self._user_mods): + if user_mods is not None: + if os.path.isabs(user_mods): + user_mods_path = user_mods + else: + user_mods_path = self.get_value('USER_MODS_DIR') + user_mods_path = os.path.join(user_mods_path, user_mods) + CIME.user_mod_support.apply_user_mods(self._caseroot, user_mods_path) + + + def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime_output_root=None): if cime_output_root is None: From 8c2bd5118c85290b87fe6a1a16708e1d23847be9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 13 Apr 2017 14:18:17 -0600 Subject: [PATCH 12/21] fix pylint issue --- scripts/lib/CIME/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 6add22645dc..e1ff041ce91 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -977,7 +977,7 @@ def apply_user_mods(self, user_mods_dir=None): else: user_mods_path = self.get_value('USER_MODS_DIR') user_mods_path = os.path.join(user_mods_path, user_mods) - CIME.user_mod_support.apply_user_mods(self._caseroot, user_mods_path) + apply_user_mods(self._caseroot, user_mods_path) From a1656f4a9721db253e7af3197adafc04274385a5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 13 Apr 2017 14:30:15 -0600 Subject: [PATCH 13/21] fix no-match args --- scripts/lib/CIME/XML/compsets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index db1c4c62d78..09048b0e148 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -27,7 +27,9 @@ def get_compset_match(self, name): nodes = self.get_nodes("compset") alias = None lname = None + science_support = [] + for node in nodes: alias = self.get_element_text("alias",root=node) lname = self.get_element_text("lname",root=node) @@ -40,7 +42,7 @@ def get_compset_match(self, name): logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support, user_mods) - return (None, None, [False]) + return (None, None, [False], None) def get_compset_var_settings(self, compset, grid): ''' From 582e308c8c789da1edbd20a61e0af8e51d42605e Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Fri, 14 Apr 2017 14:14:12 -0600 Subject: [PATCH 14/21] first changes needed for aquaplanet som --- config/cesm/config_grids.xml | 25 ++++++++++++++++--- config/xml_schemas/config_grids_v2.xsd | 2 +- .../docn/cime_config/config_component.xml | 7 ++++-- .../cime_config/namelist_definition_docn.xml | 4 +++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/config/cesm/config_grids.xml b/config/cesm/config_grids.xml index 269839e58c7..8fab255610d 100644 --- a/config/cesm/config_grids.xml +++ b/config/cesm/config_grids.xml @@ -361,13 +361,20 @@ gx1v7 - + 0.9x1.25 0.9x1.25 0.9x1.25 gx1v6 + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + null + + 0.9x1.25 0.9x1.25 @@ -506,6 +513,13 @@ gx1v6 + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + null + + 1.9x2.5 1.9x2.5 @@ -959,9 +973,11 @@ 288 192 domain.lnd.fv0.9x1.25_gx1v6.090309.nc - domain.ocn.0.9x1.25_gx1v6_090403.nc + domain.ocn.0.9x1.25_gx1v6_090403.nc domain.lnd.fv0.9x1.25_gx1v7.151020.nc - domain.ocn.fv0.9x1.25_gx1v7.151020.nc + domain.ocn.fv0.9x1.25_gx1v7.151020.nc + /glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc + /glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc 0.9x1.25 is FV 1-deg grid: @@ -969,6 +985,7 @@ 144 96 domain.lnd.fv1.9x2.5_gx1v6.090206.nc domain.ocn.1.9x2.5_gx1v6_090403.nc + domain.aqua.fv1.9x2.5.nc 1.9x2.5 is FV 2-deg grid: @@ -1483,7 +1500,7 @@ cpl/gridmaps/gx1v7/map_gx1v7_TO_ww3a_splice_170214.nc - + cpl/gridmaps/T31/map_T31_TO_ww3a_bilin_131104.nc diff --git a/config/xml_schemas/config_grids_v2.xsd b/config/xml_schemas/config_grids_v2.xsd index 2bac0a1de03..b4680b93f9e 100644 --- a/config/xml_schemas/config_grids_v2.xsd +++ b/config/xml_schemas/config_grids_v2.xsd @@ -4,7 +4,7 @@ - + diff --git a/src/components/data_comps/docn/cime_config/config_component.xml b/src/components/data_comps/docn/cime_config/config_component.xml index 165d21c1dad..686ab1bea33 100644 --- a/src/components/data_comps/docn/cime_config/config_component.xml +++ b/src/components/data_comps/docn/cime_config/config_component.xml @@ -15,7 +15,7 @@ char - prescribed,som,copyall,interannual,null + prescribed,pres_aquap,som,som_aquap,copyall,interannual,null prescribed null @@ -24,6 +24,8 @@ us20 interannual copyall + pres_aquap + som_aquap run_component_docn env_run.xml @@ -69,7 +71,8 @@ UNSET - pop_frc.1x1d.090130.nc + pop_frc.1x1d.090130.nc + /glade/u/home/benedict/ys/datain/cesm2_0_beta03.som.forcing/cam4.som.forcing.aquaplanet.QzaFix_h30Fix_TspunFix.fv19_CTL.nc run_component_docn env_run.xml diff --git a/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 32d2d7f5ba5..5d249cdbd80 100644 --- a/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -47,7 +47,9 @@ List of streams used for the given docn_mode. prescribed + prescribed som + som interannual copyall @@ -312,7 +314,9 @@ NULL SSTDATA + SSTDATA SOM + SOM IAF COPYALL From 28e84b71dd695bc8c2b0c6a330d14cc4d698b24f Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 14 Apr 2017 20:34:34 -0600 Subject: [PATCH 15/21] Fix precedence of user_mods application Now --user-mods on the command line (including testmods) will take precedence over the user_mods set by the compset - for user_nl files, shell_commands and SourceMods. I have tested this with this diff to the A compset diff --git a/src/drivers/mct/cime_config/config_compsets.xml b/src/drivers/mct/cime_config/config_compsets.xml index c11354e..7e6c2c9 100644 --- a/src/drivers/mct/cime_config/config_compsets.xml +++ b/src/drivers/mct/cime_config/config_compsets.xml @@ -40,6 +40,7 @@ A 2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV + /Users/sacks/temporary/user_mods_compset Along with this create_newcase command: ./create_newcase -case test_0414m -compset A -res f45_g37 \ --run-unsupported \ --user-mods-dir /Users/sacks/temporary/user_mods_command_line where the contents of the two relevant user_mods directories are: --- user_mods_compset/shell_commands --- ./xmlchange STOP_N=101 --- user_mods_compset/SourceMods/src.drv/mysrc.F90 --- user_mods_compset --- user_mods_compset/user_nl_cpl --- user_mods_compset --- user_mods_command_line/shell_commands --- ./xmlchange STOP_N=102 --- user_mods_command_line/SourceMods/src.drv/mysrc.F90 --- user_mods_command_line --- user_mods_command_line/user_nl_cpl --- user_mods_command_line The final contents are: --- user_nl_cpl --- user_mods_compset user_mods_command_line --- shell_commands --- ./xmlchange --force STOP_N=102 --- SourceMods/src.drv/mysrc.F90 --- user_mods_command_line And $ ./xmlquery STOP_N STOP_N: 102 thus demonstrating that the user_mods on the command-line takes precedence over the compset's user_mods. --- scripts/lib/CIME/XML/compsets.py | 6 +++- scripts/lib/CIME/case.py | 6 ++-- scripts/lib/CIME/user_mod_support.py | 47 +++++++++++++++------------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 09048b0e148..f5b85d8a5d2 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -38,7 +38,11 @@ def get_compset_match(self, name): for node in science_support_nodes: science_support.append(node.get("grid")) - user_mods = self.get_optional_node("user_mods", root=node) + user_mods_node = self.get_optional_node("user_mods", root=node) + if user_mods_node is not None: + user_mods = user_mods_node.text + else: + user_mods = None logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support, user_mods) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index e1ff041ce91..d7e051021c0 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -969,8 +969,10 @@ def apply_user_mods(self, user_mods_dir=None): User mods can be specified on the create_newcase command line (usually when called from create test) or they can be in the compset definition, or both. """ - - for user_mods in (user_mods_dir, self._user_mods): + + # This looping order will lead to the specified user_mods_dir taking + # precedence over self._user_mods, if there are any conflicts. + for user_mods in (self._user_mods, user_mods_dir): if user_mods is not None: if os.path.isabs(user_mods): user_mods_path = user_mods diff --git a/scripts/lib/CIME/user_mod_support.py b/scripts/lib/CIME/user_mod_support.py index 572910e43b2..d27c7255542 100644 --- a/scripts/lib/CIME/user_mod_support.py +++ b/scripts/lib/CIME/user_mod_support.py @@ -14,6 +14,9 @@ def apply_user_mods(caseroot, user_mods_path): updating SourceMods and creating case shell_commands and xmlchange_cmds files First remove case shell_commands files if any already exist + + If this function is called multiple times, settings from later calls will + take precedence over earlier calls, if there are conflicts. ''' case_shell_command_files = [os.path.join(caseroot,"shell_commands"), os.path.join(caseroot,"xmlchange_cmnds")] @@ -22,6 +25,13 @@ def apply_user_mods(caseroot, user_mods_path): os.remove(shell_command_file) include_dirs = build_include_dirs_list(user_mods_path) + # If a user_mods dir 'foo' includes 'bar', the include_dirs list returned + # from build_include_dirs has 'foo' before 'bar'. But with the below code, + # directories that occur later in the list take precedence over the earlier + # ones, and we want 'foo' to take precedence over 'bar' in this case (in + # general: we want a given user_mods directory to take precedence over any + # mods that it includes). So we reverse include_dirs to accomplish this. + include_dirs.reverse() logger.debug("include_dirs are %s"%include_dirs) for include_dir in include_dirs: # write user_nl_xxx file in caseroot @@ -31,7 +41,11 @@ def apply_user_mods(caseroot, user_mods_path): if len(newcontents) == 0: continue case_user_nl = user_nl.replace(include_dir, caseroot) - update_user_nl_file(case_user_nl, newcontents) + # If the same variable is set twice in a user_nl file, the later one + # takes precedence. So by appending the new contents, later entries + # in the include_dirs list take precedence over earlier entries. + with open(case_user_nl, "a") as fd: + fd.write(newcontents) # update SourceMods in caseroot for root, _, files in os.walk(include_dir,followlinks=True,topdown=False): @@ -39,21 +53,18 @@ def apply_user_mods(caseroot, user_mods_path): for sfile in files: source_mods = os.path.join(root,sfile) case_source_mods = source_mods.replace(include_dir, caseroot) + # We overwrite any existing SourceMods file so that later + # include_dirs take precedence over earlier ones if os.path.isfile(case_source_mods): - logger.warn("Refusing to overwrite existing SourceMods in %s"%case_source_mods) + logger.warn("WARNING: Overwriting existing SourceMods in %s"%case_source_mods) else: logger.info("Adding SourceMod to case %s"%case_source_mods) - try: - shutil.copyfile(source_mods, case_source_mods) - except: - expect(False, "Could not write file %s in caseroot %s" - %(case_source_mods,caseroot)) + try: + shutil.copyfile(source_mods, case_source_mods) + except: + expect(False, "Could not write file %s in caseroot %s" + %(case_source_mods,caseroot)) - # Reverse include_dirs to make sure xmlchange commands are called in the - # correct order; it may be desireable to reverse include_dirs above the - # previous loop and then append user_nl changes rather than prepend them. - include_dirs.reverse() - for include_dir in include_dirs: # create xmlchange_cmnds and shell_commands in caseroot shell_command_files = glob.glob(os.path.join(include_dir,"shell_commands")) +\ glob.glob(os.path.join(include_dir,"xmlchange_cmnds")) @@ -70,6 +81,8 @@ def apply_user_mods(caseroot, user_mods_path): shell_commands_file) with open(shell_commands_file,"r") as fd: new_shell_commands = fd.read().replace("xmlchange","xmlchange --force") + # By appending the new commands to the end, settings from later + # include_dirs take precedence over earlier ones with open(case_shell_commands, "a") as fd: fd.write(new_shell_commands) @@ -78,16 +91,6 @@ def apply_user_mods(caseroot, user_mods_path): os.chmod(shell_command_file, 0777) run_cmd_no_fail(shell_command_file) -def update_user_nl_file(case_user_nl, contents): - if os.path.isfile(case_user_nl): - with open(case_user_nl, "r") as fd: - old_contents = fd.read() - contents = contents + old_contents - logger.debug("Pre-pending file %s"%(case_user_nl)) - with open(case_user_nl, "w") as fd: - fd.write(contents) - - def build_include_dirs_list(user_mods_path, include_dirs=None): ''' From 4df9f7eb0a1d6ff151c6bbc79793d6ef0eb546ab Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Sat, 15 Apr 2017 11:04:56 -0600 Subject: [PATCH 16/21] fixed issue with using new user_mods element in compset definition --- scripts/lib/CIME/XML/compsets.py | 2 -- scripts/lib/CIME/case.py | 9 +++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index f5b85d8a5d2..2d96aaed3e6 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -37,13 +37,11 @@ def get_compset_match(self, name): science_support_nodes = self.get_nodes("science_support", root=node) for node in science_support_nodes: science_support.append(node.get("grid")) - user_mods_node = self.get_optional_node("user_mods", root=node) if user_mods_node is not None: user_mods = user_mods_node.text else: user_mods = None - logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) return (lname, alias, science_support, user_mods) return (None, None, [False], None) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index d7e051021c0..38ef14a3e46 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -964,12 +964,16 @@ def create_caseroot(self, clone=False): self._create_caseroot_sourcemods() self._create_caseroot_tools() + # Apply user_mods if part of compset + if self._user_mods is not None: + self._user_mods = self.get_resolved_value(self._user_mods) + self.apply_user_mods() + def apply_user_mods(self, user_mods_dir=None): """ User mods can be specified on the create_newcase command line (usually when called from create test) or they can be in the compset definition, or both. """ - # This looping order will lead to the specified user_mods_dir taking # precedence over self._user_mods, if there are any conflicts. for user_mods in (self._user_mods, user_mods_dir): @@ -981,9 +985,6 @@ def apply_user_mods(self, user_mods_dir=None): user_mods_path = os.path.join(user_mods_path, user_mods) apply_user_mods(self._caseroot, user_mods_path) - - - def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime_output_root=None): if cime_output_root is None: cime_output_root = self.get_value("CIME_OUTPUT_ROOT") From 4f6bb99fef2410c246e1c93b621f4ff98550d890 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 16 Apr 2017 06:12:03 -0600 Subject: [PATCH 17/21] Add unit tests for user_mod_support --- .../lib/CIME/tests/test_user_mod_support.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 scripts/lib/CIME/tests/test_user_mod_support.py diff --git a/scripts/lib/CIME/tests/test_user_mod_support.py b/scripts/lib/CIME/tests/test_user_mod_support.py new file mode 100644 index 00000000000..c8e582f01b1 --- /dev/null +++ b/scripts/lib/CIME/tests/test_user_mod_support.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +import unittest +import shutil +import tempfile +import os +from CIME.user_mod_support import apply_user_mods + +# ======================================================================== +# Define some parameters +# ======================================================================== + +_SOURCEMODS = os.path.join("SourceMods", "src.drv") + +class TestUserModSupport(unittest.TestCase): + + # ======================================================================== + # Test helper functions + # ======================================================================== + + def setUp(self): + self._caseroot = tempfile.mkdtemp() + self._caseroot_sourcemods = os.path.join(self._caseroot, _SOURCEMODS) + os.makedirs(self._caseroot_sourcemods) + self._user_mods_parent_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._caseroot, ignore_errors=True) + shutil.rmtree(self._user_mods_parent_dir, ignore_errors=True) + + def createUserMod(self, name, include_dirs=None): + """Create a user_mods directory with the given name. + + This directory is created within self._user_mods_parent_dir + + For name='foo', it will contain: + + - A user_nl_cpl file with contents: + foo + + - A shell_commands file with contents: + echo foo >> /PATH/TO/CASEROOT/shell_commands_result + + - A file in _SOURCEMODS named myfile.F90 with contents: + foo + + If include_dirs is given, it should be a list of strings, giving names + of other user_mods directories to include. e.g., if include_dirs is + ['foo1', 'foo2'], then this will create a file 'include_user_mods' that + contains paths to the 'foo1' and 'foo2' user_mods directories, one per + line. + """ + + mod_dir = os.path.join(self._user_mods_parent_dir, name) + os.makedirs(mod_dir) + mod_dir_sourcemods = os.path.join(mod_dir, _SOURCEMODS) + os.makedirs(mod_dir_sourcemods) + + with open(os.path.join(mod_dir, "user_nl_cpl"), "w") as user_nl_cpl: + user_nl_cpl.write(name + "\n") + with open(os.path.join(mod_dir, "shell_commands"), "w") as shell_commands: + command = "echo %s >> %s/shell_commands_result\n"%(name, + self._caseroot) + shell_commands.write(command) + with open(os.path.join(mod_dir_sourcemods, "myfile.F90"), "w") as f90_file: + f90_file.write(name + "\n") + + if include_dirs: + with open(os.path.join(mod_dir, "include_user_mods"), "w") as include_user_mods: + for one_include in include_dirs: + include_user_mods.write(os.path.join(self._user_mods_parent_dir, one_include) + "\n") + + def assertResults(self, expected_user_nl_cpl, + expected_shell_commands_result, + expected_sourcemod, + msg = ""): + """Asserts that the contents of the files in self._caseroot match expectations + + If msg is provided, it is printed for some failing assertions + """ + + path_to_user_nl_cpl = os.path.join(self._caseroot, "user_nl_cpl") + self.assertTrue(os.path.isfile(path_to_user_nl_cpl), + msg = msg + ": user_nl_cpl does not exist") + with open(path_to_user_nl_cpl, "r") as user_nl_cpl: + contents = user_nl_cpl.read() + self.assertEqual(expected_user_nl_cpl, contents) + + path_to_shell_commands_result = os.path.join(self._caseroot, "shell_commands_result") + self.assertTrue(os.path.isfile(path_to_shell_commands_result), + msg = msg + ": shell_commands_result does not exist") + with open(path_to_shell_commands_result, "r") as shell_commands_result: + contents = shell_commands_result.read() + self.assertEqual(expected_shell_commands_result, contents) + + path_to_sourcemod = os.path.join(self._caseroot_sourcemods, "myfile.F90") + self.assertTrue(os.path.isfile(path_to_sourcemod), + msg = msg + ": sourcemod file does not exist") + with open(path_to_sourcemod, "r") as sourcemod: + contents = sourcemod.read() + self.assertEqual(expected_sourcemod, contents) + + # ======================================================================== + # Begin actual tests + # ======================================================================== + + def test_basic(self): + self.createUserMod("foo") + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo")) + self.assertResults(expected_user_nl_cpl = "foo\n", + expected_shell_commands_result = "foo\n", + expected_sourcemod = "foo\n", + msg = "test_basic") + + def test_two_applications(self): + """If apply_user_mods is called twice, the second should appear after the first so that it takes precedence.""" + + self.createUserMod("foo1") + self.createUserMod("foo2") + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo1")) + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo2")) + self.assertResults(expected_user_nl_cpl = "foo1\nfoo2\n", + expected_shell_commands_result = "foo1\nfoo2\n", + expected_sourcemod = "foo2\n", + msg = "test_two_applications") + + def test_include(self): + """If there is an included mod, the main one should appear after the included one so that it takes precedence.""" + + self.createUserMod("base") + self.createUserMod("derived", include_dirs=["base"]) + + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "derived")) + + self.assertResults(expected_user_nl_cpl = "base\nderived\n", + expected_shell_commands_result = "base\nderived\n", + expected_sourcemod = "derived\n", + msg = "test_include") + + def test_duplicate_includes(self): + """Test multiple includes, where both include the same base mod. + + The base mod should only be included once. + """ + + self.createUserMod("base") + self.createUserMod("derived1", include_dirs=["base"]) + self.createUserMod("derived2", include_dirs=["base"]) + self.createUserMod("derived_combo", + include_dirs = ["derived1", "derived2"]) + + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "derived_combo")) + + # NOTE(wjs, 2017-04-15) The ordering of derived1 vs. derived2 is not + # critical here: If this aspect of the behavior changes, the + # expected_contents can be changed to match the new behavior in this + # respect. + expected_contents = """base +derived2 +derived1 +derived_combo +""" + self.assertResults(expected_user_nl_cpl = expected_contents, + expected_shell_commands_result = expected_contents, + expected_sourcemod = "derived_combo\n", + msg = "test_duplicate_includes") From fc4ac0b488c178b85097a779fcf5c8c8c7f473ba Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 16 Apr 2017 20:40:27 -0600 Subject: [PATCH 18/21] fix typo --- scripts/lib/CIME/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 38ef14a3e46..8532bf928c8 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -96,7 +96,7 @@ def __init__(self, case_root=None, read_only=True): self._component_classes = [] self._is_env_loaded = False # these are user_mods as defined in the compset - # Command Line user_mods are handeled seperately + # Command Line user_mods are handled seperately self._user_mods = None self.thread_count = None self.total_tasks = None From 50e5d7a609ae4abf5fd1589d1ef147c10d1b7f2a Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 16 Apr 2017 21:20:55 -0600 Subject: [PATCH 19/21] Fix timing of applying compset user_mods The previous implementation had two problems: 1. If you specified a user_mods on the command-line along with a compset that has its own user_mods, then the compset's user_mods get applied twice. 2. The new place where there was a call to apply_user_mods happened too early: xmlchange commands can not be done at that point. This fixes these problems. I have tested this with the same changes described in 9cc7740ca19723a3a7b056f0e63e7abc28450132. I tested create_newcase with no user_mods, user_mods just from the command line, user_mods just from the compset, and user_mods from the command line and the compset. --- scripts/create_newcase | 2 +- scripts/lib/CIME/case.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/create_newcase b/scripts/create_newcase index f1877cd76e5..a28f8c8d129 100755 --- a/scripts/create_newcase +++ b/scripts/create_newcase @@ -195,7 +195,7 @@ def _main_func(description): if user_mods_dir is not None: if os.path.isdir(user_mods_dir): user_mods_dir = os.path.abspath(user_mods_dir) - case.apply_user_mods(user_mods_dir) + case.apply_user_mods(user_mods_dir) # Lock env_case.xml lock_file("env_case.xml", caseroot) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 8532bf928c8..71432f0b840 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -964,19 +964,20 @@ def create_caseroot(self, clone=False): self._create_caseroot_sourcemods() self._create_caseroot_tools() - # Apply user_mods if part of compset - if self._user_mods is not None: - self._user_mods = self.get_resolved_value(self._user_mods) - self.apply_user_mods() - def apply_user_mods(self, user_mods_dir=None): """ User mods can be specified on the create_newcase command line (usually when called from create test) or they can be in the compset definition, or both. """ + + if self._user_mods is None: + compset_user_mods_resolved = None + else: + compset_user_mods_resolved = self.get_resolved_value(self._user_mods) + # This looping order will lead to the specified user_mods_dir taking # precedence over self._user_mods, if there are any conflicts. - for user_mods in (self._user_mods, user_mods_dir): + for user_mods in (compset_user_mods_resolved, user_mods_dir): if user_mods is not None: if os.path.isabs(user_mods): user_mods_path = user_mods From 30755760ebcff3c002ad53251cc1c6f4e525ac6a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Apr 2017 09:43:01 -0600 Subject: [PATCH 20/21] add comment to README and stdout --- scripts/lib/CIME/case.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 71432f0b840..6015b9aa6ed 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -960,6 +960,10 @@ def create_caseroot(self, clone=False): comp_grid = "%s_GRID"%component_class append_status("%s is %s"%(comp_grid,self.get_value(comp_grid)), "README.case", caseroot=self._caseroot) + if self._user_mods is not None: + note = "This compset includes user_mods %s"%self._user_mods + append_status(note, "README.case", caseroot=self._caseroot) + logger.info(note) if not clone: self._create_caseroot_sourcemods() self._create_caseroot_tools() From 0266e9d994a44cc9f6a4b91659c71c8dca5f16b1 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 17 Apr 2017 10:05:16 -0600 Subject: [PATCH 21/21] Print user_mods directory upfront Point is: I want to make it hard to miss --- scripts/lib/CIME/case.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 597876a4e8a..351e6f4a43a 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -437,7 +437,10 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf self.set_lookup_value("USER_MODS_DIR" , user_mods_dir) self.set_lookup_value("PES_SPEC_FILE" , files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False)) - logger.info("Compset longname is %s " %(match)) + compset_info = "Compset longname is %s"%(match) + if self._user_mods is not None: + compset_info += " with user_mods directory %s"%(self._user_mods) + logger.info(compset_info) logger.info("Compset specification file is %s" %(compsets_filename)) logger.info("Pes specification file is %s" %(pesfile)) return compset_alias, science_support @@ -935,7 +938,10 @@ def create_caseroot(self, clone=False): # Open a new README.case file in $self._caseroot append_status(" ".join(sys.argv), "README.case", caseroot=self._caseroot) - append_status("Compset longname is %s"%self.get_value("COMPSET"), + compset_info = "Compset longname is %s"%(self.get_value("COMPSET")) + if self._user_mods is not None: + compset_info += " with user_mods directory %s"%(self._user_mods) + append_status(compset_info, "README.case", caseroot=self._caseroot) append_status("Compset specification file is %s" % (self.get_value("COMPSETS_SPEC_FILE")), @@ -949,10 +955,6 @@ def create_caseroot(self, clone=False): comp_grid = "%s_GRID"%component_class append_status("%s is %s"%(comp_grid,self.get_value(comp_grid)), "README.case", caseroot=self._caseroot) - if self._user_mods is not None: - note = "This compset includes user_mods %s"%self._user_mods - append_status(note, "README.case", caseroot=self._caseroot) - logger.info(note) if not clone: self._create_caseroot_sourcemods() self._create_caseroot_tools()