From 1402f21d52220eaa3959e8a3ecefa526bfb43bd5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 2 May 2017 09:44:24 -0600 Subject: [PATCH 1/6] remove empty mpi-serial settings from config_machines.xml --- config/acme/machines/config_machines.xml | 86 ++++++----------------- config/cesm/machines/config_machines.xml | 67 ++++-------------- scripts/lib/CIME/XML/env_mach_specific.py | 6 +- 3 files changed, 42 insertions(+), 117 deletions(-) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 403042d37106..5fcf03e8322a 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -61,7 +61,7 @@ edison acme_developer intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH}/acme_scratch/edison $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -185,7 +185,7 @@ cori acme_developer intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH}/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -318,7 +318,7 @@ cori-knl-haswell-is-default acme_developer intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH}/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -453,7 +453,7 @@ acme_developer Darwin gnu - openmpi,mpich,mpi-serial + openmpi,mpich $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld $ENV{HOME}/projects/acme/cesm-inputdata $ENV{HOME}/projects/acme/ptclm-data @@ -476,7 +476,7 @@ LINUX acme_developer gnu - openmpi,mpich,mpi-serial + openmpi,mpich $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld $ENV{HOME}/projects/acme/cesm-inputdata @@ -506,7 +506,7 @@ acme_developer LINUX gnu,intel - openmpi,mpi-serial + openmpi $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -562,7 +562,7 @@ acme_developer LINUX gnu,intel - openmpi,mpi-serial + openmpi $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -624,7 +624,7 @@ acme_developer LINUX gnu - mpich,mpi-serial + mpich $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -675,7 +675,7 @@ wwwproxy.sandia.gov:80 acme_integration intel - openmpi,mpi-serial + openmpi LINUX /gscratch/$USER/acme_scratch/skybridge $CIME_OUTPUT_ROOT/$CASE/run @@ -703,9 +703,6 @@ --npernode $PES_PER_NODE - - - /usr/share/Modules/init/python.py /usr/share/Modules/init/perl.pm @@ -754,7 +751,7 @@ wwwproxy.sandia.gov:80 acme_integration intel - openmpi,mpi-serial + openmpi LINUX /gscratch/$USER/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run @@ -782,9 +779,6 @@ --npernode $PES_PER_NODE - - - /usr/share/Modules/init/python.py /usr/share/Modules/init/perl.pm @@ -832,7 +826,7 @@ b.*.lcrc.anl.gov acme_integration gnu,pgi,intel,nag - mvapich,mpich,openmpi,mpi-serial + mvapich,mpich,openmpi /lcrc/project/$PROJECT/$USER/acme_scratch /lcrc/project/$PROJECT $CIME_OUTPUT_ROOT/$CASE/run @@ -856,9 +850,6 @@ -n $TOTALPES - - - /etc/profile.d/a_softenv.csh /etc/profile.d/a_softenv.sh @@ -907,7 +898,7 @@ b.*.lcrc.anl.gov acme_integration intel,gnu,pgi - mvapich,openmpi,mpi-serial + mvapich,openmpi /lcrc/group/acme/$USER/acme_scratch /lcrc/group/acme $CIME_OUTPUT_ROOT/$CASE/run @@ -938,9 +929,6 @@ -n $TOTALPES - - - /etc/profile.d/a_softenv.csh /etc/profile.d/a_softenv.sh @@ -1062,7 +1050,7 @@ LLNL Linux Cluster, Linux (pgi), 12 pes/node, batch system is Moab intel, pgi - mpich,mpi-serial + mpich /p/lscratche/$CCSMUSER/ACME/$CASE/run /p/lscratche/$CCSMUSER/$CASE/bld /p/lscratche/$USER @@ -1092,9 +1080,6 @@ -N {{ num_nodes }} - - - @@ -1201,7 +1186,7 @@ PNL Haswell cluster, OS is Linux, batch system is SLURM LINUX intel,pgi - mpich,mpi-serial + mpich constance /pic/scratch/$CCSMUSER/csmruns/$CASE/run /pic/scratch/$CCSMUSER/csmruns/$CASE/bld @@ -1225,10 +1210,6 @@ --kill-on-bad-exit - - - - @@ -1236,7 +1217,7 @@ oic2 acme_developer gnu - mpich,mpi-serial,openmpi + mpich,openmpi /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld /home/$USER/models/ACME @@ -1256,9 +1237,6 @@ --hostfile $ENV{PBS_NODEFILE} - - - /usr/share/Modules/init/sh /usr/share/Modules/init/csh @@ -1284,7 +1262,7 @@ oic5 acme_developer gnu - mpich,mpi-serial,openmpi + mpich,openmpi /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld /home/$USER/models/ACME @@ -1304,9 +1282,6 @@ --hostfile $ENV{PBS_NODEFILE} - - - @@ -1318,7 +1293,7 @@ acme_developer LINUX gnu,intel - openmpi,mpi-serial + openmpi /lustre/pfs1/cades-ccsi/scratch/$USER/$CASE/run /lustre/pfs1/cades-ccsi/scratch/$USER/$CASE/bld /lustre/pfs1/cades-ccsi/proj-shared/project_acme/ACME_inputdata @@ -1338,9 +1313,6 @@ -np $TOTALPES --hostfile $ENV{PBS_NODEFILE} - - - /usr/share/Modules/init/sh @@ -1367,7 +1339,7 @@ Received node event ec_node acme_developer pgi,pgiacc,intel,cray - mpich,mpi-serial + mpich $ENV{HOME}/acme_scratch/$PROJECT $ENV{MEMBERWORK}/$PROJECT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1534,7 +1506,7 @@ eos acme_developer intel - mpich,mpi-serial + mpich $ENV{HOME}/acme_scratch/$PROJECT $ENV{MEMBERWORK}/$PROJECT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1563,9 +1535,6 @@ -cc numa_node - - - $MODULESHOME/init/sh $MODULESHOME/init/csh @@ -1687,9 +1656,6 @@ -n {{ num_tasks }} - - - 4 36 32 @@ -1703,7 +1669,7 @@ wf-fe.*.lanl.gov acme_developer intel,gnu - openmpi,mvapich,mpi-serial + openmpi,mvapich LINUX /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/run /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/bld @@ -1772,9 +1738,6 @@ -n $TOTALPES - - - 4 16 16 @@ -1788,7 +1751,7 @@ .*yellowstone LINUX intel,pgi,gnu - mpich2,pempi,mpi-serial + mpich2,pempi /glade/scratch/$USER $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1807,9 +1770,6 @@ mpirun.lsf - - - /glade/apps/opt/lmod/lmod/init/perl /glade/apps/opt/lmod/lmod/init/csh @@ -1958,7 +1918,7 @@ Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX intel - openmpi,mpi-serial + openmpi /global/scratch/$ENV{USER} $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -2020,7 +1980,7 @@ Lawrencium LR2 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX intel - openmpi,mpi-serial + openmpi /global/scratch/$ENV{USER} $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index a982a5a0f9b2..b6da2472d0e3 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -55,7 +55,7 @@ h2o CNL pgi,cray,gnu - mpich,mpi-serial + mpich /scratch/sciteam/$USER $ENV{CESMDATAROOT}/inputdata $ENV{CESMDATAROOT}/inputdata/atm/datm7 @@ -134,7 +134,7 @@ Brutus Linux Cluster ETH (pgi(9.0-1)/intel(10.1.018) with openi(1.4.1)/mvapich2(1.4rc2), 16 pes/node, batch system LSF, added by UB LINUX pgi,intel - openmpi,mpich,mpi-serial + openmpi,mpich /cluster/work/uwis/$USER /cluster/work/uwis/ccsm/inputdata /cluster/work/uwis/ccsm/inputdata/atm/datm7 @@ -159,9 +159,6 @@ - - - /etc/profile.d/modules.perl /etc/profile.d/modules.sh @@ -228,9 +225,6 @@ requires some workarounds; these workarounds are implemented here --> /opt/sgi/mpt/mpt-2.15/bin/mpirun $ENV{UNIT_TEST_HOST} -np 1 - - - /glade/u/apps/ch/opt/lmod/7.2.1/lmod/lmod/init/perl /glade/u/apps/ch/opt/lmod/7.2.1/lmod/lmod/init/env_modules_python.py @@ -305,9 +299,6 @@ 24 24 FALSE - - - srun @@ -392,7 +383,7 @@ cori CNL intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH} /project/projectdirs/ccsm1/inputdata /project/projectdirs/ccsm1/inputdata/atm/datm7 @@ -495,7 +486,7 @@ NERSC XC* KNL, os is CNL, 68 pes/node, batch system is Slurm CNL intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH} /project/projectdirs/ccsm1/inputdata /project/projectdirs/ccsm1/inputdata/atm/datm7 @@ -742,7 +733,7 @@ edison CNL intel,gnu,cray - mpt,mpi-serial + mpt $ENV{CSCRATCH} /project/projectdirs/ccsm1/inputdata /project/projectdirs/ccsm1/inputdata/atm/datm7 @@ -908,7 +899,7 @@ NOAA XE6, os is CNL, 24 pes/node, batch system is PBS CNL pgi - mpich,mpi-serial + mpich /lustre/fs/scratch/Julio.T.Bacmeister /lustre/fs/scratch/Julio.T.Bacmeister/inputdata /lustre/fs/scratch/Julio.T.Bacmeister/inputdata @@ -930,9 +921,6 @@ -d $ENV{OMP_NUM_THREADS} - - - /opt/modules/default/init/perl.pm /opt/modules/default/init/csh @@ -997,9 +985,6 @@ omplace - - - /picnic/u/apps/la/opt/lmod/6.5/gnu/4.8.5/lmod/lmod/init/perl /picnic/u/apps/la/opt/lmod/6.5/gnu/4.8.5/lmod/lmod/init/env_modules_python.py @@ -1044,7 +1029,7 @@ LINUX sonproxy.sandia.gov:80 gnu - openmpi,mpi-serial + openmpi /sems-data-store/ACME/timings $ENV{HOME}/acme/scratch /sems-data-store/ACME/inputdata @@ -1099,7 +1084,7 @@ LINUX wwwproxy.sandia.gov:80 gnu - openmpi,mpi-serial + openmpi /sems-data-store/ACME/timings $ENV{HOME}/acme/scratch /sems-data-store/ACME/inputdata @@ -1215,9 +1200,6 @@ 32 32 FALSE - - - mpiexec_mpt @@ -1250,7 +1232,7 @@ NASA/AMES Linux Cluster, Linux (ia64), 2.5 GHz Haswell Intel Xeon E5-2680v3 processors, 24 pes/node (two 12-core processors) and 128 GB of memory/node, batch system is PBS LINUX intel - mpich,mpi-serial + mpich /nobackup/$USER /nobackup/mjmills2/ccsmdata/inputdata /nobackup/mjmills2/ccsmdata/inputdata/atm/datm7 @@ -1263,9 +1245,6 @@ 24 24 TRUE - - - mpiexec_mpt @@ -1302,7 +1281,7 @@ NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.93 GHz Westmere processors, 12 pes/node and 24 GB of memory, batch system is PBS LINUX intel - mpich,mpi-serial + mpich /nobackup/$USER /nobackup/mjmills2/ccsmdata/inputdata /nobackup/mjmills2/ccsmdata/inputdata/atm/datm7 @@ -1315,9 +1294,6 @@ 12 12 TRUE - - - mpiexec_mpt @@ -1354,7 +1330,7 @@ NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.6 GHz Sandy Bridge processors, 16 cores/node and 32 GB of memory, batch system is PBS LINUX intel - mpich,mpi-serial + mpich /nobackup/$USER /nobackup/mjmills2/ccsmdata/inputdata /nobackup/mjmills2/ccsmdata/inputdata/atm/datm7 @@ -1367,9 +1343,6 @@ 16 16 TRUE - - - mpiexec_mpt @@ -1406,7 +1379,7 @@ NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.8 GHz Ivy Bridge processors, 20 cores/node and 3.2 GB of memory per core, batch system is PBS LINUX intel - mpich,mpi-serial + mpich /nobackup/$USER /nobackup/mjmills2/ccsmdata/inputdata /nobackup/mjmills2/ccsmdata/inputdata/atm/datm7 @@ -1419,9 +1392,6 @@ 20 20 TRUE - - - mpiexec_mpt @@ -1458,7 +1428,7 @@ CSCS Cray XE6, os is CNL, 32 pes/node, batch system is SLURM CNL pgi,cray,gnu - mpich,mpi-serial + mpich /scratch/rosa/$USER /project/s433/cesm_inputdata /project/s433/cesm_inputdata/atm/datm7 @@ -1491,7 +1461,7 @@ LINUX wwwproxy.sandia.gov:80 intel - openmpi,mpi-serial + openmpi /projects/ccsm/timings /gscratch/$USER/acme_scratch/skybridge /projects/ccsm/inputdata @@ -1514,9 +1484,6 @@ -npernode $PES_PER_NODE - - - /usr/share/Modules/init/python.py /usr/share/Modules/init/perl.pm @@ -1577,9 +1544,6 @@ ibrun - - - /opt/apps/lmod/lmod/init/perl /opt/apps/lmod/lmod/init/env_modules_python.py @@ -1640,9 +1604,6 @@ 30 15 TRUE - - - diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py index 284bb933b27b..e49e2b81a804 100644 --- a/scripts/lib/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -355,7 +355,7 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only if xml_attrib == value: matches += 1 - elif key == "mpilib" and xml_attrib == "default": + elif key == "mpilib" and value != "mpi-serial" and xml_attrib == "default": is_default = True else: all_match = False @@ -374,6 +374,10 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only best_match = mpirun_node best_num_matched = matches + # if there are no special arguments required for mpi-serial it need not have an entry in config_machines.xml + if "mpilib" in attribs and attribs["mpilib"] == "mpi-serial" and best_match is None: + return "",[] + expect(best_match is not None or default_match is not None, "Could not find a matching MPI for attributes: %s" % attribs) From 8d3def3f57bf72a33e8f0dcd0f28798c56accf50 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 2 May 2017 14:10:46 -0600 Subject: [PATCH 2/6] add error for machines that do not support mpi-serial --- config/acme/machines/config_machines.xml | 6 ++++++ config/cesm/machines/config_machines.xml | 3 +++ scripts/lib/CIME/XML/env_mach_specific.py | 4 +++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 5fcf03e8322a..b907705018f3 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -1028,6 +1028,9 @@ --envs OMP_NUM_THREADS=$OMP_NUM_THREADS + + unsupported + /etc/profile.d/00softenv.csh soft @@ -1121,6 +1124,9 @@ --envs OMP_NUM_THREADS=$OMP_NUM_THREADS + + unsupported + /etc/profile.d/00softenv.csh soft diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index b6da2472d0e3..43f8ded78b03 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -1163,6 +1163,9 @@ --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} + + unsupported + /etc/profile.d/00softenv.csh /etc/profile.d/00softenv.sh diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py index e49e2b81a804..a30cf28e07c4 100644 --- a/scripts/lib/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -399,5 +399,7 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only exec_node = self.get_node("executable", root=the_match) expect(exec_node is not None,"No executable found") executable = exec_node.text - + expect("unsupported" not in executable, "%s is not supported for this machine"%attribs["mpilib"]) + + return executable, args From c230cdeb472d634ada563e0d67c10e9674193155 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 3 May 2017 11:29:56 -0600 Subject: [PATCH 3/6] add back mpi-serial for acme --- config/acme/machines/config_machines.xml | 66 +++++++++++++----------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index b907705018f3..bbef132d2b90 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -61,7 +61,7 @@ edison acme_developer intel,gnu,cray - mpt + mpt,mpi-serial $ENV{SCRATCH}/acme_scratch/edison $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -185,7 +185,7 @@ cori acme_developer intel,gnu,cray - mpt + mpt,mpi-serial $ENV{SCRATCH}/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -318,7 +318,7 @@ cori-knl-haswell-is-default acme_developer intel,gnu,cray - mpt + mpt,mpi-serial $ENV{SCRATCH}/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -453,7 +453,7 @@ acme_developer Darwin gnu - openmpi,mpich + openmpi,mpich,mpi-serial $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld $ENV{HOME}/projects/acme/cesm-inputdata $ENV{HOME}/projects/acme/ptclm-data @@ -476,7 +476,7 @@ LINUX acme_developer gnu - openmpi,mpich + openmpi,mpich,mpi-serial $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld $ENV{HOME}/projects/acme/cesm-inputdata @@ -506,7 +506,7 @@ acme_developer LINUX gnu,intel - openmpi + openmpi,mpi-serial $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -562,7 +562,7 @@ acme_developer LINUX gnu,intel - openmpi + openmpi,mpi-serial $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -624,7 +624,7 @@ acme_developer LINUX gnu - mpich + mpich,mpi-serial $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -675,7 +675,7 @@ wwwproxy.sandia.gov:80 acme_integration intel - openmpi + openmpi,mpi-serial LINUX /gscratch/$USER/acme_scratch/skybridge $CIME_OUTPUT_ROOT/$CASE/run @@ -751,7 +751,7 @@ wwwproxy.sandia.gov:80 acme_integration intel - openmpi + openmpi,mpi-serial LINUX /gscratch/$USER/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run @@ -826,7 +826,7 @@ b.*.lcrc.anl.gov acme_integration gnu,pgi,intel,nag - mvapich,mpich,openmpi + mvapich,mpich,openmpi,mpi-serial /lcrc/project/$PROJECT/$USER/acme_scratch /lcrc/project/$PROJECT $CIME_OUTPUT_ROOT/$CASE/run @@ -898,7 +898,7 @@ b.*.lcrc.anl.gov acme_integration intel,gnu,pgi - mvapich,openmpi + mvapich,openmpi,mpi-serial /lcrc/group/acme/$USER/acme_scratch /lcrc/group/acme $CIME_OUTPUT_ROOT/$CASE/run @@ -1028,9 +1028,9 @@ --envs OMP_NUM_THREADS=$OMP_NUM_THREADS - - unsupported - + + unsupported + /etc/profile.d/00softenv.csh soft @@ -1053,7 +1053,7 @@ LLNL Linux Cluster, Linux (pgi), 12 pes/node, batch system is Moab intel, pgi - mpich + mpich,mpi-serial /p/lscratche/$CCSMUSER/ACME/$CASE/run /p/lscratche/$CCSMUSER/$CASE/bld /p/lscratche/$USER @@ -1124,9 +1124,9 @@ --envs OMP_NUM_THREADS=$OMP_NUM_THREADS - - unsupported - + + unsupported + /etc/profile.d/00softenv.csh soft @@ -1192,7 +1192,7 @@ PNL Haswell cluster, OS is Linux, batch system is SLURM LINUX intel,pgi - mpich + mpich,mpi-serial constance /pic/scratch/$CCSMUSER/csmruns/$CASE/run /pic/scratch/$CCSMUSER/csmruns/$CASE/bld @@ -1216,6 +1216,7 @@ --kill-on-bad-exit + @@ -1223,7 +1224,7 @@ oic2 acme_developer gnu - mpich,openmpi + mpich,mpi-serial,openmpi /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld /home/$USER/models/ACME @@ -1268,7 +1269,7 @@ oic5 acme_developer gnu - mpich,openmpi + mpich,mpi-serial,openmpi /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld /home/$USER/models/ACME @@ -1299,7 +1300,7 @@ acme_developer LINUX gnu,intel - openmpi + openmpi,mpi-serial /lustre/pfs1/cades-ccsi/scratch/$USER/$CASE/run /lustre/pfs1/cades-ccsi/scratch/$USER/$CASE/bld /lustre/pfs1/cades-ccsi/proj-shared/project_acme/ACME_inputdata @@ -1345,7 +1346,7 @@ Received node event ec_node acme_developer pgi,pgiacc,intel,cray - mpich + mpich,mpi-serial $ENV{HOME}/acme_scratch/$PROJECT $ENV{MEMBERWORK}/$PROJECT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1512,7 +1513,7 @@ eos acme_developer intel - mpich + mpich,mpi-serial $ENV{HOME}/acme_scratch/$PROJECT $ENV{MEMBERWORK}/$PROJECT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1541,6 +1542,7 @@ -cc numa_node + $MODULESHOME/init/sh $MODULESHOME/init/csh @@ -1590,7 +1592,7 @@ 64M - + @@ -1619,7 +1621,7 @@ /usr/share/lmod/lmod/libexec/lmod python module module - + /usr/projects/climate/SHARED_CLIMATE/modulefiles/all @@ -1662,6 +1664,7 @@ -n {{ num_tasks }} + 4 36 32 @@ -1675,7 +1678,7 @@ wf-fe.*.lanl.gov acme_developer intel,gnu - openmpi,mvapich + openmpi,mvapich,mpi-serial LINUX /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/run /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/bld @@ -1744,6 +1747,7 @@ -n $TOTALPES + 4 16 16 @@ -1757,7 +1761,7 @@ .*yellowstone LINUX intel,pgi,gnu - mpich2,pempi + mpich2,pempi,mpi-serial /glade/scratch/$USER $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1924,7 +1928,7 @@ Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX intel - openmpi + openmpi,mpi-serial /global/scratch/$ENV{USER} $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1986,7 +1990,7 @@ Lawrencium LR2 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX intel - openmpi + openmpi,mpi-serial /global/scratch/$ENV{USER} $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld From e90430a8f172cda0ac6871cc7dfbb0c6e7495086 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 3 May 2017 12:29:28 -0600 Subject: [PATCH 4/6] force acme to specify mpi-serial --- scripts/lib/CIME/XML/machines.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index 5806045608ad..37c1d13e0a10 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -256,7 +256,9 @@ def is_valid_MPIlib(self, mpilib, attributes=None): >>> machobj.is_valid_MPIlib("fake-mpi") False """ - return mpilib == "mpi-serial" or self.get_field_from_list("MPILIBS", reqval=mpilib, attributes=attributes) is not None + model = CIME.utils.get_model() + return (model == "cesm" and mpilib == "mpi-serial") or \ + self.get_field_from_list("MPILIBS", reqval=mpilib, attributes=attributes) is not None def has_batch_system(self): """ From 91150def0034ee8f50543872bc0d75ba9825e7b5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 3 May 2017 12:37:48 -0600 Subject: [PATCH 5/6] fix unit test --- scripts/lib/CIME/XML/machines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index 37c1d13e0a10..0dcabc9e4e69 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -4,7 +4,7 @@ from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files -from CIME.utils import convert_to_unknown_type +from CIME.utils import convert_to_unknown_type, get_model import socket @@ -256,7 +256,7 @@ def is_valid_MPIlib(self, mpilib, attributes=None): >>> machobj.is_valid_MPIlib("fake-mpi") False """ - model = CIME.utils.get_model() + model = get_model() return (model == "cesm" and mpilib == "mpi-serial") or \ self.get_field_from_list("MPILIBS", reqval=mpilib, attributes=attributes) is not None From 3b9e81541165a03cf9e006dd1dd95bc3046a4cb6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 3 May 2017 15:40:42 -0600 Subject: [PATCH 6/6] result of negotiation --- config/acme/machines/config_machines.xml | 52 ++++++++++------------- config/cesm/machines/config_machines.xml | 3 -- scripts/lib/CIME/XML/env_mach_specific.py | 4 +- scripts/lib/CIME/XML/machines.py | 5 +-- 4 files changed, 26 insertions(+), 38 deletions(-) diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index bbef132d2b90..6dfa110dbb2d 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -61,7 +61,7 @@ edison acme_developer intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH}/acme_scratch/edison $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -185,7 +185,7 @@ cori acme_developer intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH}/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -318,7 +318,7 @@ cori-knl-haswell-is-default acme_developer intel,gnu,cray - mpt,mpi-serial + mpt $ENV{SCRATCH}/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -453,7 +453,7 @@ acme_developer Darwin gnu - openmpi,mpich,mpi-serial + openmpi,mpich $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld $ENV{HOME}/projects/acme/cesm-inputdata $ENV{HOME}/projects/acme/ptclm-data @@ -476,7 +476,7 @@ LINUX acme_developer gnu - openmpi,mpich,mpi-serial + openmpi,mpich $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld $ENV{HOME}/projects/acme/cesm-inputdata @@ -506,7 +506,7 @@ acme_developer LINUX gnu,intel - openmpi,mpi-serial + openmpi $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -562,7 +562,7 @@ acme_developer LINUX gnu,intel - openmpi,mpi-serial + openmpi $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -624,7 +624,7 @@ acme_developer LINUX gnu - mpich,mpi-serial + mpich $ENV{HOME}/acme/scratch $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -675,7 +675,7 @@ wwwproxy.sandia.gov:80 acme_integration intel - openmpi,mpi-serial + openmpi LINUX /gscratch/$USER/acme_scratch/skybridge $CIME_OUTPUT_ROOT/$CASE/run @@ -751,7 +751,7 @@ wwwproxy.sandia.gov:80 acme_integration intel - openmpi,mpi-serial + openmpi LINUX /gscratch/$USER/acme_scratch $CIME_OUTPUT_ROOT/$CASE/run @@ -826,7 +826,7 @@ b.*.lcrc.anl.gov acme_integration gnu,pgi,intel,nag - mvapich,mpich,openmpi,mpi-serial + mvapich,mpich,openmpi /lcrc/project/$PROJECT/$USER/acme_scratch /lcrc/project/$PROJECT $CIME_OUTPUT_ROOT/$CASE/run @@ -898,7 +898,7 @@ b.*.lcrc.anl.gov acme_integration intel,gnu,pgi - mvapich,openmpi,mpi-serial + mvapich,openmpi /lcrc/group/acme/$USER/acme_scratch /lcrc/group/acme $CIME_OUTPUT_ROOT/$CASE/run @@ -1028,9 +1028,6 @@ --envs OMP_NUM_THREADS=$OMP_NUM_THREADS - - unsupported - /etc/profile.d/00softenv.csh soft @@ -1053,7 +1050,7 @@ LLNL Linux Cluster, Linux (pgi), 12 pes/node, batch system is Moab intel, pgi - mpich,mpi-serial + mpich /p/lscratche/$CCSMUSER/ACME/$CASE/run /p/lscratche/$CCSMUSER/$CASE/bld /p/lscratche/$USER @@ -1124,9 +1121,6 @@ --envs OMP_NUM_THREADS=$OMP_NUM_THREADS - - unsupported - /etc/profile.d/00softenv.csh soft @@ -1192,7 +1186,7 @@ PNL Haswell cluster, OS is Linux, batch system is SLURM LINUX intel,pgi - mpich,mpi-serial + mpich constance /pic/scratch/$CCSMUSER/csmruns/$CASE/run /pic/scratch/$CCSMUSER/csmruns/$CASE/bld @@ -1224,7 +1218,7 @@ oic2 acme_developer gnu - mpich,mpi-serial,openmpi + mpich,openmpi /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld /home/$USER/models/ACME @@ -1269,7 +1263,7 @@ oic5 acme_developer gnu - mpich,mpi-serial,openmpi + mpich,openmpi /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld /home/$USER/models/ACME @@ -1300,7 +1294,7 @@ acme_developer LINUX gnu,intel - openmpi,mpi-serial + openmpi /lustre/pfs1/cades-ccsi/scratch/$USER/$CASE/run /lustre/pfs1/cades-ccsi/scratch/$USER/$CASE/bld /lustre/pfs1/cades-ccsi/proj-shared/project_acme/ACME_inputdata @@ -1346,7 +1340,7 @@ Received node event ec_node acme_developer pgi,pgiacc,intel,cray - mpich,mpi-serial + mpich $ENV{HOME}/acme_scratch/$PROJECT $ENV{MEMBERWORK}/$PROJECT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1513,7 +1507,7 @@ eos acme_developer intel - mpich,mpi-serial + mpich $ENV{HOME}/acme_scratch/$PROJECT $ENV{MEMBERWORK}/$PROJECT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1678,7 +1672,7 @@ wf-fe.*.lanl.gov acme_developer intel,gnu - openmpi,mvapich,mpi-serial + openmpi,mvapich LINUX /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/run /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/bld @@ -1761,7 +1755,7 @@ .*yellowstone LINUX intel,pgi,gnu - mpich2,pempi,mpi-serial + mpich2,pempi /glade/scratch/$USER $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1928,7 +1922,7 @@ Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX intel - openmpi,mpi-serial + openmpi /global/scratch/$ENV{USER} $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld @@ -1990,7 +1984,7 @@ Lawrencium LR2 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX intel - openmpi,mpi-serial + openmpi /global/scratch/$ENV{USER} $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 43f8ded78b03..b6da2472d0e3 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -1163,9 +1163,6 @@ --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} - - unsupported - /etc/profile.d/00softenv.csh /etc/profile.d/00softenv.sh diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py index a30cf28e07c4..e49e2b81a804 100644 --- a/scripts/lib/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -399,7 +399,5 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only exec_node = self.get_node("executable", root=the_match) expect(exec_node is not None,"No executable found") executable = exec_node.text - expect("unsupported" not in executable, "%s is not supported for this machine"%attribs["mpilib"]) - - + return executable, args diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index 0dcabc9e4e69..67e37b6e9734 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -4,7 +4,7 @@ from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files -from CIME.utils import convert_to_unknown_type, get_model +from CIME.utils import convert_to_unknown_type import socket @@ -256,8 +256,7 @@ def is_valid_MPIlib(self, mpilib, attributes=None): >>> machobj.is_valid_MPIlib("fake-mpi") False """ - model = get_model() - return (model == "cesm" and mpilib == "mpi-serial") or \ + return mpilib == "mpi-serial" or \ self.get_field_from_list("MPILIBS", reqval=mpilib, attributes=attributes) is not None def has_batch_system(self):