Skip to content

Commit

Permalink
get mira working again
Browse files Browse the repository at this point in the history
  • Loading branch information
@jedwards4b
jedwards4b committed Jul 13, 2016
1 parent 0f298cf commit 2a5e2b0
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 62 deletions.
49 changes: 14 additions & 35 deletions cime_config/cesm/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@
<arg flag="-n" name="$TOTALPES/$PES_PER_NODE"/>
<arg flag="-q" name="$JOB_QUEUE"/>
<arg flag="--mode script"/>
</submit_args>
<arg flag="--env" name='"CIMEROOT=$CIMEROOT"'/>
</submit_args>
</batch_system>

<batch_system type="lsf" version="9.1">
Expand Down Expand Up @@ -142,21 +143,21 @@
</batch_system>

<!-- babbage is PBS -->
<batch_system MACH="babbage" version="x.y">
<batch_system MACH="babbage" type="pbs">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>

<!-- babbageKnc is PBS -->
<batch_system MACH="babbageKnc" version="x.y">
<batch_system MACH="babbageKnc" type="pbs">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>

<!-- brutus is PBS -->
<batch_system type="pbs" MACH="brutus" version="x.y">
<batch_system type="pbs" MACH="brutus" >
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
Expand Down Expand Up @@ -186,25 +187,25 @@
</batch_system>

<!-- brutus is PBS -->
<batch_system MACH="brutus" version="x.y">
<batch_system MACH="brutus" type="pbs">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>

<!-- eos is PBS -->
<batch_system MACH="eos" version="x.y">
<!-- eos is PBS -->
<batch_system MACH="eos" type="pbs">
<jobid_pattern>^(\d+)</jobid_pattern>
<directives>
<directive>-A $PROJECT</directive>
<directive>-l mppwidth={{ mppwidth }}</directive>
<directive>-l nodes={{ num_nodes }}</directive>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>
</batch_system>

<!-- erebus is PBS -->
<batch_system MACH="erebus" version="x.y">
<batch_system MACH="erebus" type="pbs">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
Expand Down Expand Up @@ -232,14 +233,6 @@
</walltimes>
</batch_system>

<!-- goldbach is PBS -->
<batch_system MACH="goldbach" version="x.y">
<directives>
<directive>-l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}</directive>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>

<!-- hobart is PBS -->
<batch_system type="pbs" MACH="hobart" version="x.y">
<directives>
Expand All @@ -253,28 +246,14 @@
</batch_system>

<!-- hera is SLURM -->
<batch_system MACH="hera">
<batch_system MACH="hera" type="slurm">
<batch_directive>#MSUB</batch_directive>
<directives>
<directive>-A ees</directive>
<directive>-l gres=lscratchd</directive>
</directives>
</batch_system>

<!-- hopper is PBS -->
<batch_system MACH="hopper" version="x.y">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>

<!-- janus is PBS -->
<batch_system MACH="janus" version="x.y">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
</batch_system>

<batch_system MACH="mira" type="cobalt">
<queues>
<queue waltimemin="0" walltimemax="360" jobmin="512" jobmax="4096" default="true">default</queue>
Expand Down Expand Up @@ -361,7 +340,7 @@
</batch_system>

<!-- all pleiades machines are PBS -->
<batch_system MACH="pleiades-wes" version="x.y">
<batch_system MACH="pleiades-wes" version="x.y" type="pbs">
<jobid_pattern>^(\S+)</jobid_pattern>
<directives>
<directive>-W group_list=$PROJECT </directive>
Expand All @@ -372,7 +351,7 @@
</batch_system>

<!-- sierra is SLURM -->
<batch_system MACH="sierra">
<batch_system MACH="sierra" type="slurm">
<batch_directive>#MSUB</batch_directive>
<directives>
<directive> </directive>
Expand Down Expand Up @@ -431,7 +410,7 @@
</batch_system>

<!-- titan is PBS -->
<batch_system MACH="titan" version="x.y">
<batch_system MACH="titan" type="pbs">
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
</directives>
Expand Down
9 changes: 4 additions & 5 deletions cime_config/cesm/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -789,18 +789,17 @@
<PES_PER_NODE>8</PES_PER_NODE>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>/usr/bin/runjob</executable>
<executable>/usr/bin/runjob</executable>
<arguments>
<arg name="label"> --label short</arg>
<!-- Ranks per node!! -->
<arg name="tasks_per_node"> -p {{ tasks_per_node }}</arg>
<arg name="tasks_per_node"> --ranks-per-node $PES_PER_NODE</arg>
<!-- Total MPI Tasks -->
<arg name="num_tasks"> -n {{ num_tasks }}</arg>
<arg name="locargs"> $LOCARGS</arg>
<arg name="num_tasks"> --np $TOTALPES</arg>
<arg name="locargs">--block $COBALT_PARTNAME --envs OMP_WAIT_POLICY=active --envs BG_SMP_FAST_WAKEUP=yes $LOCARGS</arg>
<arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
<arg name="omp_stacksize"> --envs OMP_STACKSIZE=32M</arg>
<arg name="thread_count"> --envs OMP_NUM_THREADS={{ thread_count }}</arg>
<arg name="colon">:</arg>
</arguments>
</mpirun>
<module_system type="soft">
Expand Down
5 changes: 4 additions & 1 deletion utils/python/CIME/XML/machines.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,10 @@ def get_full_mpirun(self, check_members, case, job):
executable, args = self.get_mpirun(mpi_attribs, check_members, case, job)

mpi_arg_string = " ".join(args.values())

batch_system = self.get_value("BATCH_SYSTEM")
if batch_system == "cobalt":
mpi_arg_string += " : "

return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, default_run_suffix)

def print_values(self):
Expand Down
26 changes: 13 additions & 13 deletions utils/python/CIME/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from CIME.XML.env_mach_specific import EnvMachSpecific
from CIME.preview_namelists import preview_namelists
from CIME.check_input_data import check_input_data

import glob, shutil, time, threading, gzip

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -94,10 +93,11 @@ def build_model(case, build_threaded, exeroot, clm_config_opts, incroot, complis
file_build = os.path.join(exeroot, "%s.bldlog.%s" % (cime_model, lid))

config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config")
stat = run_cmd("%s/buildexe %s %s %s >> %s 2>&1" %
(config_dir, caseroot, bldroot, libroot, file_build),
from_dir=bldroot, ok_to_fail=True, verbose=True)[0]

f = open(file_build, "w")
stat = run_cmd("%s/buildexe %s %s %s" %
(config_dir, caseroot, bldroot, libroot),
from_dir=bldroot, ok_to_fail=True, verbose=True, arg_stdout=f)[0]
f.close()
expect(stat == 0, "ERROR: buildexe failed, cat %s" % file_build)

# Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
Expand Down Expand Up @@ -493,14 +493,13 @@ def build_libraries(case, exeroot, caseroot, cimeroot, libroot, mpilib, lid, mac
os.makedirs(full_lib_path)

file_build = os.path.join(sharedpath, "%s.bldlog.%s" % (lib, lid))
my_file = os.path.join(os.path.dirname(machines_file), "buildlib.%s" % lib)
with open(file_build, "w") as fd:
fd.write("Current env:\n%s" % "\n".join([" %s = %s" % (env, os.environ[env]) for env in sorted(os.environ)]))

my_file = os.path.join(os.path.dirname(machines_file), "buildlib.%s" % lib)
stat = run_cmd("%s %s %s >> %s 2>&1" %
(my_file, sharedpath, caseroot, file_build),
stat = run_cmd("%s %s %s" %
(my_file, sharedpath, caseroot),
from_dir=exeroot,
ok_to_fail=True, verbose=True)[0]
ok_to_fail=True, verbose=True, arg_stdout=fd)[0]
expect(stat == 0, "ERROR: buildlib.%s failed, cat %s" % (lib, file_build))
logs.append(file_build)

Expand Down Expand Up @@ -528,10 +527,11 @@ def build_libraries(case, exeroot, caseroot, cimeroot, libroot, mpilib, lid, mac
def _build_model_thread(config_dir, compclass, caseroot, bldroot, libroot, incroot, file_build,
thread_bad_results):
###############################################################################
with open(file_build, "w") as fd:
stat = run_cmd("%s/buildlib %s %s %s " %
(config_dir, caseroot, bldroot, libroot),
from_dir=bldroot, ok_to_fail=True,verbose=True, arg_stdout=fd)[0]

stat = run_cmd("%s/buildlib %s %s %s >> %s 2>&1" %
(config_dir, caseroot, bldroot, libroot, file_build),
from_dir=bldroot, ok_to_fail=True,verbose=True)[0]
if (stat != 0):
thread_bad_results.append("ERROR: %s.buildlib failed, see %s" % (compclass, file_build))

Expand Down
12 changes: 4 additions & 8 deletions utils/python/CIME/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ def run_cmd(cmd, ok_to_fail=False, input_str=None, from_dir=None, verbose=None,
if (arg_stdout is _hack):
arg_stdout = subprocess.PIPE
if (arg_stderr is _hack):
arg_stderr = subprocess.PIPE
arg_stderr = subprocess.STDOUT


if (verbose or logger.level == logging.DEBUG):
logger.info("RUN: %s" % cmd)
Expand Down Expand Up @@ -198,13 +199,8 @@ def run_cmd(cmd, ok_to_fail=False, input_str=None, from_dir=None, verbose=None,
if (ok_to_fail):
return stat, output, errput
else:
if (arg_stderr is not None):
errput = errput if errput is not None else open(arg_stderr.name, "r").read()
expect(stat == 0, "Command: '%s' failed with error '%s'%s" %
(cmd, errput, "" if from_dir is None else " from dir '%s'" % from_dir))
else:
expect(stat == 0, "Command: '%s' failed%s. See terminal output" %
(cmd, "" if from_dir is None else " from dir '%s'" % from_dir))
expect(stat == 0, "Command: '%s' failed%s. See terminal output" %
(cmd, "" if from_dir is None else " from dir '%s'" % from_dir))
return output

def check_minimum_python_version(major, minor):
Expand Down

0 comments on commit 2a5e2b0

Please sign in to comment.