Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add comment pes file #2315

Merged
merged 5 commits into from
Feb 23, 2018
Merged

Add comment pes file #2315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f1346cf
pass the comment from config_pes to env_mach_pes
jedwards4b Feb 23, 2018
39d868a
all tests passing
jedwards4b Feb 23, 2018
cb3d102
cleanup;
jedwards4b Feb 23, 2018
31163b7
response to review
jedwards4b Feb 23, 2018
1ac2fb6
fix pylint issue
jedwards4b Feb 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions config/xml_schemas/env_mach_pes.xsd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
<xs:attribute name="version" type="xs:decimal"/>

<!-- simple elements -->
<xs:element name="comment" type="xs:string"/>
<xs:element name="header" type="xs:string"/>
<xs:element name="type" type="xs:NCName"/>
<xs:element name="valid_values" type="xs:string"/>
Expand All @@ -16,6 +17,7 @@
<xs:complexType>
<xs:sequence>
<xs:element ref="header"/>
<xs:element ref="comment" minOccurs="0" maxOccurs="1"/>
<xs:element maxOccurs="unbounded" ref="group"/>
</xs:sequence>
<xs:attribute ref="id" use="required" />
Expand Down
8 changes: 8 additions & 0 deletions scripts/lib/CIME/XML/env_mach_pes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ def __init__(self, case_root=None, infile="env_mach_pes.xml", components=None):
schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_mach_pes.xsd")
EnvBase.__init__(self, case_root, infile, schema=schema)

def add_comment(self, comment):
if comment is not None:
node = self.make_child("comment", text=comment)
# make_child adds to the end of the file but we want it to follow the header
# so we need to remove it and add it in the correct position
self.remove_child(node)
self.add_child(node, position=1)

def get_value(self, vid, attribute=None, resolved=True, subgroup=None, max_mpitasks_per_node=None): # pylint: disable=arguments-differ
# Special variable NINST_MAX is used to determine the number of
# drivers in multi-driver mode.
Expand Down
19 changes: 9 additions & 10 deletions scripts/lib/CIME/XML/generic_xml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,26 +44,22 @@ def __init__(self, infile=None, schema=None, root_name_override=None, root_attri
self.root = None
self.locked = False
self.read_only = read_only

if infile == None:
# if file is not defined just return
self.filename = None
self.filename = infile
if infile is None:
return

if os.path.isfile(infile) and os.access(infile, os.R_OK):
# If file is defined and exists, read it
self.filename = infile
self.read(infile, schema)
else:
# if file does not exist create a root xml element
# and set it's id to file
expect(not read_only, "Makes no sense to have empty read-only file")

expect(not self.read_only, "Makes no sense to have empty read-only file")
logger.debug("File {} does not exists.".format(infile))
expect("$" not in infile,"File path not fully resolved {}".format(infile))

self.filename = infile
root = _Element(ET.Element("xml"))

if root_name_override:
self.root = self.make_child(root_name_override, root=root, attributes=root_attrib_override)
else:
Expand Down Expand Up @@ -170,13 +166,16 @@ def name(self, node):
def text(self, node):
return node.xml_element.text

def add_child(self, node, root=None):
def add_child(self, node, root=None, position=None):
"""
Add element node to self at root
"""
expect(not self.locked and not self.read_only, "locked")
root = root if root is not None else self.root
root.xml_element.append(node.xml_element)
if position is not None:
root.xml_element.insert(position, node.xml_element)
else:
root.xml_element.append(node.xml_element)

def copy(self, node):
return deepcopy(node)
Expand Down
28 changes: 21 additions & 7 deletions scripts/lib/CIME/XML/pes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts='M', mpilib=None):
oother_settings = {}
other_settings = {}
o_grid_nodes = []
comments = None
# Get any override nodes
overrides = self.get_optional_child("overrides")
ocomments = None
if overrides is not None:
o_grid_nodes = self.get_children("grid", root = overrides)
opes_ntasks, opes_nthrds, opes_rootpe, opes_pstrid, oother_settings = self._find_matches(o_grid_nodes, grid, compset, machine, pesize_opts, True)
opes_ntasks, opes_nthrds, opes_rootpe, opes_pstrid, oother_settings, ocomments = self._find_matches(o_grid_nodes, grid, compset, machine, pesize_opts, True)

# Get all the nodes
grid_nodes = self.get_children("grid")
if o_grid_nodes:
Expand All @@ -42,12 +45,15 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts='M', mpilib=None):
grid_nodes = list(gn_set)


pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings = self._find_matches(grid_nodes, grid, compset, machine, pesize_opts, False)
pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings, comments = self._find_matches(grid_nodes, grid, compset, machine, pesize_opts, False)
pes_ntasks.update(opes_ntasks)
pes_nthrds.update(opes_nthrds)
pes_rootpe.update(opes_rootpe)
pes_pstrid.update(opes_pstrid)
other_settings.update(oother_settings)
if ocomments is not None:
comments = ocomments


if mpilib == "mpi-serial":
for i in iter(pes_ntasks):
Expand All @@ -64,7 +70,10 @@ def find_pes_layout(self, grid, compset, machine, pesize_opts='M', mpilib=None):
logger.info("Pes setting: rootpe is {} ".format(pes_rootpe))
logger.info("Pes setting: pstrid is {} ".format(pes_pstrid))
logger.info("Pes other settings: {}".format(other_settings))
return pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings
if comments is not None:
logger.info("Pes comments: {}".format(comments))

return pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings, comments

def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, override=False):
grid_choice = None
Expand All @@ -74,6 +83,7 @@ def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, overrid
max_points = -1
pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings = {}, {}, {}, {}, {}
pe_select = None
comment = None
for grid_node in grid_nodes:
grid_match = self.get(grid_node, "name")
if grid_match == "any" or re.search(grid_match,grid):
Expand All @@ -96,7 +106,9 @@ def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, overrid
for node in self.get_children(root=pes_node):
vid = self.name(node)
logger.info("vid is {}".format(vid))
if "ntasks" in vid:
if "comment" in vid:
comment = self.text(node)
elif "ntasks" in vid:
for child in self.get_children(root=node):
pes_ntasks[self.name(child).upper()] = int(self.text(child))
elif "nthrds" in vid:
Expand Down Expand Up @@ -131,7 +143,9 @@ def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, overrid
for node in self.get_children(root=pe_select):
vid = self.name(node)
logger.debug("vid is {}".format(vid))
if "ntasks" in vid:
if "comment" in vid:
comment = self.text(node)
elif "ntasks" in vid:
for child in self.get_children(root=node):
pes_ntasks[self.name(child).upper()] = int(self.text(child))
elif "nthrds" in vid:
Expand All @@ -144,7 +158,7 @@ def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, overrid
for child in self.get_children(root=node):
pes_pstrid[self.name(child).upper()] = int(self.text(child))
# if the value is already upper case its something else we are trying to set
elif vid == self.name(node) and vid != "comment":
elif vid == self.name(node):
other_settings[vid] = self.text(node)
if grid_choice != 'any' or logger.isEnabledFor(logging.DEBUG):
logger.info("Pes setting: grid match is {} ".format(grid_choice ))
Expand All @@ -155,4 +169,4 @@ def _find_matches(self, grid_nodes, grid, compset, machine, pesize_opts, overrid
if pesize_choice != 'any' or logger.isEnabledFor(logging.DEBUG):
logger.info("Pes setting: pesize match is {} ".format(pesize_choice))

return pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings
return pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other_settings, comment
5 changes: 3 additions & 2 deletions scripts/lib/CIME/case.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -671,7 +671,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib):
pes_rootpe = {}
pes_pstrid = {}
other = {}

comment = None
force_tasks = None
force_thrds = None

Expand All @@ -687,7 +687,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib):
force_tasks = int(match2.group(1))
pes_nthrds = pesobj.find_pes_layout(self._gridname, self._compsetname, machine_name, mpilib=mpilib)[1]
else:
pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other = pesobj.find_pes_layout(self._gridname, self._compsetname,
pes_ntasks, pes_nthrds, pes_rootpe, pes_pstrid, other, comment = pesobj.find_pes_layout(self._gridname, self._compsetname,
machine_name, pesize_opts=pecount, mpilib=mpilib)

if match1 or match2:
Expand All @@ -705,6 +705,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib):
pes_rootpe[string_] = 0

mach_pes_obj = self.get_env("mach_pes")
mach_pes_obj.add_comment(comment)

if other is not None:
for key, value in other.items():
Expand Down