Skip to content

Commit

Permalink
Merge pull request #1 from apcraig/mvertens/nuopcB3
Browse files Browse the repository at this point in the history 
auto generation of run time sequence
  • Loading branch information
@apcraig
apcraig authored Jul 25, 2017
2 parents c80ba6d + 655e675 commit 467f615
Show file tree
Hide file tree
Showing 55 changed files with 24,549 additions and 779 deletions.
26 changes: 21 additions & 5 deletions config/cesm/config_files.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
<values>
<value component="allactive">$SRCROOT/cime_config/config_compsets.xml</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/config_compsets.xml</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/config_compsets.xml</value>
<value component="cam" >$SRCROOT/components/cam/cime_config/config_compsets.xml</value>
<value component="cism" >$SRCROOT/components/cism/cime_config/config_compsets.xml</value>
<value component="clm" >$SRCROOT/components/clm/cime_config/config_compsets.xml</value>
Expand All @@ -106,6 +107,7 @@
<values>
<value component="allactive">$SRCROOT/cime_config/config_pes.xml</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/config_pes.xml</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/config_pes.xml</value>
<value component="cam" >$SRCROOT/components/cam/cime_config/config_pes.xml</value>
<value component="cism" >$SRCROOT/components/cism/cime_config/config_pes.xml</value>
<value component="clm" >$SRCROOT/components/clm/cime_config/config_pes.xml</value>
Expand Down Expand Up @@ -167,6 +169,7 @@
<values>
<value component="allactive">$SRCROOT/cime_config/testlist_allactive.xml</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/testdefs/testlist_drv.xml</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/testdefs/testlist_drv.xml</value>
<value component="cam" >$SRCROOT/components/cam/cime_config/testdefs/testlist_cam.xml</value>
<value component="cism" >$SRCROOT/components/cism/cime_config/testdefs/testlist_cism.xml</value>
<value component="clm" >$SRCROOT/components/clm/cime_config/testdefs/testlist_clm.xml</value>
Expand All @@ -187,6 +190,7 @@
<values>
<value component="allactive">$SRCROOT/cime_config/testmods_dirs</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/testdefs/testmods_dirs</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/testdefs/testmods_dirs</value>
<value component="cam" >$SRCROOT/components/cam/cime_config/testdefs/testmods_dirs</value>
<value component="cism" >$SRCROOT/components/cism/cime_config/testdefs/testmods_dirs</value>
<value component="clm" >$SRCROOT/components/clm/cime_config/testdefs/testmods_dirs</value>
Expand All @@ -206,6 +210,7 @@
<values>
<value component="allactive">$SRCROOT/cime_config/usermods_dirs</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/usermods_dirs</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/usermods_dirs</value>
<value component="cam" >$SRCROOT/components/cam/cime_config/usermods_dirs</value>
<value component="cism" >$SRCROOT/components/cism/cime_config/usermods_dirs</value>
<value component="clm" >$SRCROOT/components/clm/cime_config/usermods_dirs</value>
Expand All @@ -225,9 +230,12 @@
<type>char</type>
<default_value>unset</default_value>
<values>
<value component="modelio" >$CIMEROOT/src/drivers/mct/cime_config/namelist_definition_modelio.xml</value>
<value component="drv_flds">$CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv.xml</value>
<value component="modelio" >$CIMEROOT/src/drivers/mct/cime_config/namelist_definition_modelio.xml</value>
<value component="modelio-nuopc" >$CIMEROOT/src/drivers/nuopc/cime_config/namelist_definition_modelio.xml</value>
<value component="drv_flds" >$CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml</value>
<value component="drv_flds-nuopc" >$CIMEROOT/src/drivers/nuopc/cime_config/namelist_definition_drv_flds.xml</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv.xml</value>
<value component="drv-nuopc" >$CIMEROOT/src/drivers/nuopc/cime_config/namelist_definition_drv.xml</value>
<!-- data model components -->
<value component="drof">$CIMEROOT/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml</value>
<value component="datm">$CIMEROOT/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml</value>
Expand Down Expand Up @@ -257,7 +265,11 @@

<entry id="CONFIG_CPL_FILE">
<type>char</type>
<default_value>$CIMEROOT/src/drivers/mct/cime_config/config_component.xml</default_value>
<default_value>unset</default_value>
<values>
<value component="drv">$CIMEROOT/src/drivers/mct/cime_config/config_component.xml</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/config_component.xml</value>
</values>
<group>case_last</group>
<file>env_case.xml</file>
<desc>file containing all non-component specific case configuration variables (for documentation only - DO NOT EDIT)</desc>
Expand All @@ -266,7 +278,11 @@

<entry id="CONFIG_CPL_FILE_MODEL_SPECIFIC">
<type>char</type>
<default_value>$CIMEROOT/src/drivers/mct/cime_config/config_component_$MODEL.xml</default_value>
<default_value>unset</default_value>
<values>
<value component="drv">$CIMEROOT/src/drivers/mct/cime_config/config_component_$MODEL.xml</value>
<value component="drv-nuopc">$CIMEROOT/src/drivers/nuopc/cime_config/config_component_$MODEL.xml</value>
</values>
<group>case_last</group>
<file>env_case.xml</file>
<desc>file containing all component specific driver configuration variables (for documentation only - DO NOT EDIT)</desc>
Expand Down
1 change: 1 addition & 0 deletions config/cesm/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,7 @@
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">64M</env>
<env name="ESMFMKFILE">/home/fischer/ESMF_code/ESMF_7_1_0_beta/lib/libg/Linux.intel.64.mvapich2.default/esmf.mk</env>
</environment_variables>
</machine>

Expand Down
15 changes: 12 additions & 3 deletions scripts/create_newcase
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ OR
help="(required) Specify a model grid resolution. "
"To see list of current compsets, use the utility query_config in this directory")

parser.add_argument("--driver", "-driver", default="mct", choices=('mct','nuopc'),
help="Specify the cime driver "
"Allowed values are ('mct','nuopc') ")

parser.add_argument("--machine", "-mach",
help="Specify a machine. default: match NODENAME_REGEX in config_machines.xml "
"To see list of current machines, use the utility query_config in this directory"
Expand Down Expand Up @@ -154,12 +158,17 @@ OR
if args.input_dir is not None:
args.input_dir = os.path.abspath(args.input_dir)

if args.driver == 'mct':
args.driver = 'drv'
if args.driver == 'nuopc':
args.driver = 'drv-nuopc'

return args.case, args.compset, args.res, args.machine, args.compiler,\
args.mpilib, args.project, args.pecount, \
args.user_mods_dir, args.user_compset, args.pesfile, \
args.user_grid, args.gridfile, args.srcroot, args.test, args.ninst, \
args.walltime, args.queue, args.output_root, args.script_root, \
run_unsupported, args.answer, args.input_dir
run_unsupported, args.answer, args.input_dir, args.driver

###############################################################################
def _main_func(description):
Expand All @@ -171,7 +180,7 @@ def _main_func(description):
user_mods_dir, user_compset, pesfile, \
user_grid, gridfile, srcroot, test, ninst, walltime, queue, \
output_root, script_root, run_unsupported, \
answer, input_dir = parse_command_line(sys.argv, cimeroot, description)
answer, input_dir, driver = parse_command_line(sys.argv, cimeroot, description)

if script_root is None:
caseroot = os.path.abspath(casename)
Expand All @@ -196,7 +205,7 @@ def _main_func(description):
user_grid=user_grid, gridfile=gridfile, ninst=ninst, test=test,
walltime=walltime, queue=queue, output_root=output_root,
run_unsupported=run_unsupported, answer=answer,
input_dir=input_dir)
input_dir=input_dir, driver=driver)

case.create_caseroot()

Expand Down
21 changes: 16 additions & 5 deletions scripts/create_test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,10 @@ OR
"The default is the first listing in MPILIBS in config_machines.xml")

if model == "cesm":
parser.add_argument("--driver", "-driver", default="mct", choices=('mct','nuopc'),
help="Specify the cime driver "
"Allowed values are ('mct','nuopc') ")

parser.add_argument("-c", "--compare",
help="While testing, compare baselines"
" against the given compare directory ")
Expand Down Expand Up @@ -353,11 +357,17 @@ OR
if args.input_dir is not None:
args.input_dir = os.path.abspath(args.input_dir)

if args.driver == 'mct':
args.driver = 'drv'
if args.driver == 'nuopc':
args.driver = 'drv-nuopc'

return test_names, test_extra_data, args.compiler, mach_obj.get_machine_name(), args.no_run, args.no_build, args.no_setup, args.no_batch,\
args.test_root, args.baseline_root, args.clean, baseline_cmp_name, baseline_gen_name, \
args.namelists_only, args.project, args.test_id, args.parallel_jobs, args.walltime, \
args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, \
args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib, args.input_dir
args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib, args.input_dir, \
args.driver

###############################################################################
def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost_map, wall_time, test_root):
Expand Down Expand Up @@ -463,7 +473,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost
def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs,
walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait,
force_procs, force_threads, mpilib, input_dir):
force_procs, force_threads, mpilib, input_dir, driver):
###############################################################################
impl = TestScheduler(test_names, test_data=test_data,
no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch,
Expand All @@ -476,7 +486,7 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build,
proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing,
queue=queue, allow_baseline_overwrite=allow_baseline_overwrite,
output_root=output_root, force_procs=force_procs, force_threads=force_threads,
mpilib=mpilib, input_dir=input_dir)
mpilib=mpilib, input_dir=input_dir, driver=driver)

success = impl.run_tests(wait=wait)

Expand Down Expand Up @@ -517,13 +527,14 @@ def _main_func(description):
test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, \
test_root, baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, \
project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, \
save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir \
save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, driver \
= parse_command_line(sys.argv, description)

sys.exit(create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only,
project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing,
queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir))
queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir,
driver))

###############################################################################

Expand Down
14 changes: 9 additions & 5 deletions scripts/lib/CIME/XML/namelist_definition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ def __init__(self, infile, files=None):
expect(os.path.isfile(infile), "File {} does not exist".format(infile))
super(NamelistDefinition, self).__init__(infile, schema=schema)

self._attributes = {}
self._entry_nodes = []
self._entry_ids = []
self._attributes = {}
self._valid_values = {}
self._entry_types = {}
self._group_names = {}
Expand Down Expand Up @@ -158,7 +158,9 @@ def get_value_match(self, vid, attributes=None, exact_match=True, entry_node=Non

if entry_node is None:
entry_node = self._nodes[vid]
value = super(NamelistDefinition, self).get_value_match(vid.lower(),attributes=all_attributes, exact_match=exact_match,
# value = super(NamelistDefinition, self).get_value_match(vid.lower(),attributes=all_attributes, exact_match=exact_match,
# entry_node=entry_node)
value = super(NamelistDefinition, self).get_value_match(vid,attributes=all_attributes, exact_match=exact_match,
entry_node=entry_node)
if value is None:
value = ''
Expand Down Expand Up @@ -260,7 +262,7 @@ def is_valid_value(self, name, value):
appear in the namelist (even for scalar variables, in which case the
length of the list is always 1).
"""
name = name.lower()
#name = name.lower()
# Separate into a type, optional length, and optional size.
type_, max_len, size = self.split_type_string(name)
invalid = []
Expand Down Expand Up @@ -372,7 +374,8 @@ def dict_to_namelist(self, dict_, filename=None):
variable_template = "Variable {!r} from file " + repr(str(filename))
groups = {}
for variable_name in dict_:
variable_lc = variable_name.lower()
#variable_lc = variable_name.lower()
variable_lc = variable_name
qualified_varname = get_fortran_name_only(variable_lc)
self._expect_variable_in_definition(qualified_varname, variable_template)
group_name = self.get_group(qualified_varname)
Expand Down Expand Up @@ -405,5 +408,6 @@ def get_default_value(self, item, attribute=None):
if attribute is not None:
all_attributes.update(attribute)

value = self.get_value_match(item.lower(), all_attributes, True)
#value = self.get_value_match(item.lower(), all_attributes, True)
value = self.get_value_match(item, all_attributes, True)
return self._split_defaults_text(value)
11 changes: 8 additions & 3 deletions scripts/lib/CIME/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

###############################################################################
def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler):
lid, caseroot, cimeroot, compiler, comp_interface):
###############################################################################
logs = []

Expand Down Expand Up @@ -88,7 +88,8 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
cime_model = get_model()
file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))

config_dir = os.path.join(cimeroot, "src", "drivers", "mct", "cime_config")
config_dir = os.path.join(cimeroot, "src", "drivers", comp_interface.lower(), "cime_config")

f = open(file_build, "w")
bldroot = os.path.join(exeroot, "cpl", "obj")
if not os.path.isdir(bldroot):
Expand All @@ -98,6 +99,8 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
.format(config_dir, caseroot, libroot, bldroot),
from_dir=bldroot, verbose=False, arg_stdout=f,
arg_stderr=subprocess.STDOUT)[0]
print "DEBUG: config_dir is ",config_dir

f.close()
analyze_build_log("{} exe".format(cime_model), file_build, compiler)
expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))
Expand Down Expand Up @@ -512,10 +515,12 @@ def _case_build_impl(caseroot, case, sharedlib_only, model_only):
logs = _build_libraries(case, exeroot, sharedpath, caseroot,
cimeroot, libroot, lid, compiler)

comp_interface = case.get_value("COMP_INTERFACE")

if not sharedlib_only:
os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators
logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, compiler))
lid, caseroot, cimeroot, compiler, comp_interface))

if not sharedlib_only:
# in case component build scripts updated the xml files, update the case object
Expand Down
Loading

0 comments on commit 467f615

Please sign in to comment.