Skip to content

Commit

Permalink
Merge pull request #1366 from ESMCI/mvertens/usermods_in_compset
Browse files Browse the repository at this point in the history
Mvertens/usermods in compset
  • Loading branch information
jedwards4b authored Apr 17, 2017
2 parents aaee38a + ddd2162 commit ca289d9
Show file tree
Hide file tree
Showing 10 changed files with 275 additions and 45 deletions.
25 changes: 21 additions & 4 deletions config/cesm/config_grids.xml
Original file line number Diff line number Diff line change
Expand Up @@ -361,13 +361,20 @@
<mask>gx1v7</mask>
</model_grid>

<model_grid alias="f09_f09" not_compset="_POP" >
<model_grid alias="f09_f09" not_compset="_POP">
<grid name="atm">0.9x1.25</grid>
<grid name="lnd">0.9x1.25</grid>
<grid name="ocnice">0.9x1.25</grid>
<mask>gx1v6</mask>
</model_grid>

<model_grid alias="f09_f09_mnull" compset="_DOCN%SAQUAP|DOCN%DAQUAP" >
<grid name="atm">0.9x1.25</grid>
<grid name="lnd">0.9x1.25</grid>
<grid name="ocnice">0.9x1.25</grid>
<mask>null</mask>
</model_grid>

<model_grid alias="f09_f09_mg16" not_compset="_POP" >
<grid name="atm">0.9x1.25</grid>
<grid name="lnd">0.9x1.25</grid>
Expand Down Expand Up @@ -506,6 +513,13 @@
<mask>gx1v6</mask>
</model_grid>

<model_grid alias="f19_f19_mnull" compset="_DOCN%SAQUAP|DOCN%DAQUAP" >
<grid name="atm">1.9x2.5</grid>
<grid name="lnd">1.9x2.5</grid>
<grid name="ocnice">1.9x2.5</grid>
<mask>null</mask>
</model_grid>

<model_grid alias="f19_f19_mg17" not_compset="_POP">
<grid name="atm">1.9x2.5</grid>
<grid name="lnd">1.9x2.5</grid>
Expand Down Expand Up @@ -959,16 +973,19 @@
<domain name="0.9x1.25">
<nx>288</nx> <ny>192</ny>
<file grid="atm|lnd" mask="gx1v6">domain.lnd.fv0.9x1.25_gx1v6.090309.nc</file>
<file grid="ocnice" mask="gx1v6">domain.ocn.0.9x1.25_gx1v6_090403.nc</file>
<file grid="ocnice" mask="gx1v6">domain.ocn.0.9x1.25_gx1v6_090403.nc</file>
<file grid="atm|lnd" mask="gx1v7">domain.lnd.fv0.9x1.25_gx1v7.151020.nc</file>
<file grid="ocnice" mask="gx1v7">domain.ocn.fv0.9x1.25_gx1v7.151020.nc</file>
<file grid="ocnice" mask="gx1v7">domain.ocn.fv0.9x1.25_gx1v7.151020.nc</file>
<file grid="atm|lnd" mask="null">/glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc</file>
<file grid="ocnice" mask="null">/glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc</file>
<desc>0.9x1.25 is FV 1-deg grid:</desc>
</domain>

<domain name="1.9x2.5">
<nx>144</nx> <ny>96</ny>
<file grid="atm|lnd" mask="gx1v6">domain.lnd.fv1.9x2.5_gx1v6.090206.nc</file>
<file grid="ocnice" mask="gx1v6">domain.ocn.1.9x2.5_gx1v6_090403.nc</file>
<file grid="ocnice" mask="null">domain.aqua.fv1.9x2.5.nc</file>
<desc>1.9x2.5 is FV 2-deg grid:</desc>
</domain>

Expand Down Expand Up @@ -1483,7 +1500,7 @@
<map name="ICE2WAV_SMAPNAME">cpl/gridmaps/gx1v7/map_gx1v7_TO_ww3a_splice_170214.nc</map>
</gridmap>

<gridmap atm_grid="T31" wav_grid="ww3a">
<gridmap atm_grid="48x96" wav_grid="ww3a">
<map name="ATM2WAV_SMAPNAME">cpl/gridmaps/T31/map_T31_TO_ww3a_bilin_131104.nc</map>
</gridmap>

Expand Down
2 changes: 2 additions & 0 deletions config/xml_schemas/config_compsets.xsd
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
<xs:element name="alias" type="xs:NCName"/>
<xs:element name="lname" type="xs:string"/>
<xs:element name="science_support"/>
<xs:element name="user_mods" type="xs:string"/>

<!-- complex elements -->

Expand All @@ -32,6 +33,7 @@
<xs:element ref="alias"/>
<xs:element ref="lname"/>
<xs:element ref="science_support" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="user_mods" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute ref="grid"/>
</xs:complexType>
Expand Down
2 changes: 1 addition & 1 deletion config/xml_schemas/config_grids_v2.xsd
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<xs:attribute name="version" type="xs:decimal"/>
<xs:attribute name="alias" type="xs:NMTOKEN"/>
<xs:attribute name="compset" type="xs:token"/>
<xs:attribute name="not_compset" type="xs:NMTOKEN"/>
<xs:attribute name="not_compset" type="xs:token"/>
<xs:attribute name="name" type="xs:NMTOKEN"/>
<xs:attribute name="grid" type="xs:token"/>
<xs:attribute name="lnd_mask" type="xs:NMTOKEN"/>
Expand Down
2 changes: 1 addition & 1 deletion scripts/create_newcase
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ def _main_func(description):
if user_mods_dir is not None:
if os.path.isdir(user_mods_dir):
user_mods_dir = os.path.abspath(user_mods_dir)
case.apply_user_mods(user_mods_dir)
case.apply_user_mods(user_mods_dir)

# Lock env_case.xml
lock_file("env_case.xml", caseroot)
Expand Down
12 changes: 9 additions & 3 deletions scripts/lib/CIME/XML/compsets.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,24 @@ def get_compset_match(self, name):
nodes = self.get_nodes("compset")
alias = None
lname = None

science_support = []

for node in nodes:
alias = self.get_element_text("alias",root=node)
lname = self.get_element_text("lname",root=node)
if alias == name or lname == name:
science_support_nodes = self.get_nodes("science_support", root=node)
for node in science_support_nodes:
science_support.append(node.get("grid"))

user_mods_node = self.get_optional_node("user_mods", root=node)
if user_mods_node is not None:
user_mods = user_mods_node.text
else:
user_mods = None
logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname))
return (lname, alias, science_support)
return (None, None, [False])
return (lname, alias, science_support, user_mods)
return (None, None, [False], None)

def get_compset_var_settings(self, compset, grid):
'''
Expand Down
48 changes: 36 additions & 12 deletions scripts/lib/CIME/case.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ def __init__(self, case_root=None, read_only=True):
self._components = []
self._component_classes = []
self._is_env_loaded = False

# these are user_mods as defined in the compset
# Command Line user_mods are handled seperately
self._user_mods = None
self.thread_count = None
self.total_tasks = None
self.tasks_per_node = None
Expand Down Expand Up @@ -419,7 +421,7 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf
# If the file exists, read it and see if there is a match for the compset alias or longname
if (os.path.isfile(compsets_filename)):
compsets = Compsets(compsets_filename)
match, compset_alias, science_support = compsets.get_compset_match(name=compset_name)
match, compset_alias, science_support, self._user_mods = compsets.get_compset_match(name=compset_name)
pesfile = files.get_value("PES_SPEC_FILE" , {"component":component})
if match is not None:
self._pesfile = pesfile
Expand All @@ -435,7 +437,10 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf
self.set_lookup_value("USER_MODS_DIR" , user_mods_dir)
self.set_lookup_value("PES_SPEC_FILE" ,
files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False))
logger.info("Compset longname is %s " %(match))
compset_info = "Compset longname is %s"%(match)
if self._user_mods is not None:
compset_info += " with user_mods directory %s"%(self._user_mods)
logger.info(compset_info)
logger.info("Compset specification file is %s" %(compsets_filename))
logger.info("Pes specification file is %s" %(pesfile))
return compset_alias, science_support
Expand Down Expand Up @@ -944,7 +949,10 @@ def create_caseroot(self, clone=False):

# Open a new README.case file in $self._caseroot
append_status(" ".join(sys.argv), "README.case", caseroot=self._caseroot)
append_status("Compset longname is %s"%self.get_value("COMPSET"),
compset_info = "Compset longname is %s"%(self.get_value("COMPSET"))
if self._user_mods is not None:
compset_info += " with user_mods directory %s"%(self._user_mods)
append_status(compset_info,
"README.case", caseroot=self._caseroot)
append_status("Compset specification file is %s" %
(self.get_value("COMPSETS_SPEC_FILE")),
Expand All @@ -958,19 +966,35 @@ def create_caseroot(self, clone=False):
comp_grid = "%s_GRID"%component_class
append_status("%s is %s"%(comp_grid,self.get_value(comp_grid)),
"README.case", caseroot=self._caseroot)
if self._user_mods is not None:
note = "This compset includes user_mods %s"%self._user_mods
append_status(note, "README.case", caseroot=self._caseroot)
logger.info(note)
if not clone:
self._create_caseroot_sourcemods()
self._create_caseroot_tools()

def apply_user_mods(self, user_mods_dir=None):
if user_mods_dir is not None:
if os.path.isabs(user_mods_dir):
user_mods_path = user_mods_dir
else:
user_mods_path = self.get_value('USER_MODS_DIR')
user_mods_path = os.path.join(user_mods_path, user_mods_dir)
self.set_value("USER_MODS_FULLPATH",user_mods_path)
apply_user_mods(self._caseroot, user_mods_path)
"""
User mods can be specified on the create_newcase command line (usually when called from create test)
or they can be in the compset definition, or both.
"""

if self._user_mods is None:
compset_user_mods_resolved = None
else:
compset_user_mods_resolved = self.get_resolved_value(self._user_mods)

# This looping order will lead to the specified user_mods_dir taking
# precedence over self._user_mods, if there are any conflicts.
for user_mods in (compset_user_mods_resolved, user_mods_dir):
if user_mods is not None:
if os.path.isabs(user_mods):
user_mods_path = user_mods
else:
user_mods_path = self.get_value('USER_MODS_DIR')
user_mods_path = os.path.join(user_mods_path, user_mods)
apply_user_mods(self._caseroot, user_mods_path)

def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime_output_root=None):
if cime_output_root is None:
Expand Down
171 changes: 171 additions & 0 deletions scripts/lib/CIME/tests/test_user_mod_support.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
#!/usr/bin/env python

import unittest
import shutil
import tempfile
import os
from CIME.user_mod_support import apply_user_mods

# ========================================================================
# Define some parameters
# ========================================================================

_SOURCEMODS = os.path.join("SourceMods", "src.drv")

class TestUserModSupport(unittest.TestCase):

# ========================================================================
# Test helper functions
# ========================================================================

def setUp(self):
self._caseroot = tempfile.mkdtemp()
self._caseroot_sourcemods = os.path.join(self._caseroot, _SOURCEMODS)
os.makedirs(self._caseroot_sourcemods)
self._user_mods_parent_dir = tempfile.mkdtemp()

def tearDown(self):
shutil.rmtree(self._caseroot, ignore_errors=True)
shutil.rmtree(self._user_mods_parent_dir, ignore_errors=True)

def createUserMod(self, name, include_dirs=None):
"""Create a user_mods directory with the given name.
This directory is created within self._user_mods_parent_dir
For name='foo', it will contain:
- A user_nl_cpl file with contents:
foo
- A shell_commands file with contents:
echo foo >> /PATH/TO/CASEROOT/shell_commands_result
- A file in _SOURCEMODS named myfile.F90 with contents:
foo
If include_dirs is given, it should be a list of strings, giving names
of other user_mods directories to include. e.g., if include_dirs is
['foo1', 'foo2'], then this will create a file 'include_user_mods' that
contains paths to the 'foo1' and 'foo2' user_mods directories, one per
line.
"""

mod_dir = os.path.join(self._user_mods_parent_dir, name)
os.makedirs(mod_dir)
mod_dir_sourcemods = os.path.join(mod_dir, _SOURCEMODS)
os.makedirs(mod_dir_sourcemods)

with open(os.path.join(mod_dir, "user_nl_cpl"), "w") as user_nl_cpl:
user_nl_cpl.write(name + "\n")
with open(os.path.join(mod_dir, "shell_commands"), "w") as shell_commands:
command = "echo %s >> %s/shell_commands_result\n"%(name,
self._caseroot)
shell_commands.write(command)
with open(os.path.join(mod_dir_sourcemods, "myfile.F90"), "w") as f90_file:
f90_file.write(name + "\n")

if include_dirs:
with open(os.path.join(mod_dir, "include_user_mods"), "w") as include_user_mods:
for one_include in include_dirs:
include_user_mods.write(os.path.join(self._user_mods_parent_dir, one_include) + "\n")

def assertResults(self, expected_user_nl_cpl,
expected_shell_commands_result,
expected_sourcemod,
msg = ""):
"""Asserts that the contents of the files in self._caseroot match expectations
If msg is provided, it is printed for some failing assertions
"""

path_to_user_nl_cpl = os.path.join(self._caseroot, "user_nl_cpl")
self.assertTrue(os.path.isfile(path_to_user_nl_cpl),
msg = msg + ": user_nl_cpl does not exist")
with open(path_to_user_nl_cpl, "r") as user_nl_cpl:
contents = user_nl_cpl.read()
self.assertEqual(expected_user_nl_cpl, contents)

path_to_shell_commands_result = os.path.join(self._caseroot, "shell_commands_result")
self.assertTrue(os.path.isfile(path_to_shell_commands_result),
msg = msg + ": shell_commands_result does not exist")
with open(path_to_shell_commands_result, "r") as shell_commands_result:
contents = shell_commands_result.read()
self.assertEqual(expected_shell_commands_result, contents)

path_to_sourcemod = os.path.join(self._caseroot_sourcemods, "myfile.F90")
self.assertTrue(os.path.isfile(path_to_sourcemod),
msg = msg + ": sourcemod file does not exist")
with open(path_to_sourcemod, "r") as sourcemod:
contents = sourcemod.read()
self.assertEqual(expected_sourcemod, contents)

# ========================================================================
# Begin actual tests
# ========================================================================

def test_basic(self):
self.createUserMod("foo")
apply_user_mods(self._caseroot,
os.path.join(self._user_mods_parent_dir, "foo"))
self.assertResults(expected_user_nl_cpl = "foo\n",
expected_shell_commands_result = "foo\n",
expected_sourcemod = "foo\n",
msg = "test_basic")

def test_two_applications(self):
"""If apply_user_mods is called twice, the second should appear after the first so that it takes precedence."""

self.createUserMod("foo1")
self.createUserMod("foo2")
apply_user_mods(self._caseroot,
os.path.join(self._user_mods_parent_dir, "foo1"))
apply_user_mods(self._caseroot,
os.path.join(self._user_mods_parent_dir, "foo2"))
self.assertResults(expected_user_nl_cpl = "foo1\nfoo2\n",
expected_shell_commands_result = "foo1\nfoo2\n",
expected_sourcemod = "foo2\n",
msg = "test_two_applications")

def test_include(self):
"""If there is an included mod, the main one should appear after the included one so that it takes precedence."""

self.createUserMod("base")
self.createUserMod("derived", include_dirs=["base"])

apply_user_mods(self._caseroot,
os.path.join(self._user_mods_parent_dir, "derived"))

self.assertResults(expected_user_nl_cpl = "base\nderived\n",
expected_shell_commands_result = "base\nderived\n",
expected_sourcemod = "derived\n",
msg = "test_include")

def test_duplicate_includes(self):
"""Test multiple includes, where both include the same base mod.
The base mod should only be included once.
"""

self.createUserMod("base")
self.createUserMod("derived1", include_dirs=["base"])
self.createUserMod("derived2", include_dirs=["base"])
self.createUserMod("derived_combo",
include_dirs = ["derived1", "derived2"])

apply_user_mods(self._caseroot,
os.path.join(self._user_mods_parent_dir, "derived_combo"))

# NOTE(wjs, 2017-04-15) The ordering of derived1 vs. derived2 is not
# critical here: If this aspect of the behavior changes, the
# expected_contents can be changed to match the new behavior in this
# respect.
expected_contents = """base
derived2
derived1
derived_combo
"""
self.assertResults(expected_user_nl_cpl = expected_contents,
expected_shell_commands_result = expected_contents,
expected_sourcemod = "derived_combo\n",
msg = "test_duplicate_includes")
Loading

0 comments on commit ca289d9

Please sign in to comment.