From bc9def784ff4966d46d6690b30cb211f43f9cd64 Mon Sep 17 00:00:00 2001
From: Alice Bertini <aliceb@ucar.edu>
Date: Tue, 25 Apr 2017 11:52:47 -0600
Subject: [PATCH] update cheyenne to use mpi4py/2.0.0-mpt native SGI mpt

---
 Machines/cheyenne_modules        | 7 ++++---
 Machines/machine_postprocess.xml | 6 ++----
 Templates/batch_cheyenne.tmpl    | 5 -----
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/Machines/cheyenne_modules b/Machines/cheyenne_modules
index af4b0c83..fdcfbb85 100755
--- a/Machines/cheyenne_modules
+++ b/Machines/cheyenne_modules
@@ -8,12 +8,13 @@ module restore system
 module load python/2.7.13
 module load numpy/1.12.0
 module load scipy/0.18.1
-module load impi/5.1.3.210
-module load mpi4py/2.0.0-impi
+##module load impi/5.1.3.210
+##module load mpi4py/2.0.0-impi
+module load mpi4py/2.0.0-mpt
 module load netcdf/4.4.1.1
 module load pynio/1.4.1
 module load matplotlib/2.0.0
-module load intel/16.0.3
+##module load intel/16.0.3
 module load nco/4.6.2
 module load ncl/6.3.0
 module load netcdf4-python/1.2.7
diff --git a/Machines/machine_postprocess.xml b/Machines/machine_postprocess.xml
index 40319e09..ec3a4db0 100644
--- a/Machines/machine_postprocess.xml
+++ b/Machines/machine_postprocess.xml
@@ -59,7 +59,7 @@
 
   <machine name="cheyenne" hostname="cheyenne">
     <timeseries_pes queue="regular" nodes="2" pes_per_node="36" wallclock="00:30:00">72</timeseries_pes>
-    <mpi_command>mpirun -n {{ pes }}</mpi_command>
+    <mpi_command>mpiexec_mpt dplace -s 1</mpi_command>
     <pythonpath></pythonpath>
     <f2py fcompiler="gfortran" f77exec="/glade/u/apps/ch/opt/gnu/6.2.0/bin/gfortran" 
 	  f90exec="/glade/u/apps/ch/opt/gnu/6.2.0/bin/gfortran">f2py</f2py>
@@ -77,11 +77,9 @@
       <module>module load python/2.7.13</module>
       <module>module load numpy/1.12.0</module>
       <module>module load scipy/0.18.1</module>
-      <module>module load impi/5.1.3.210</module>
-      <module>module load mpi4py/2.0.0-impi</module>
+      <module>module load mpi4py/2.0.0-mpt</module>
       <module>module load pynio/1.4.1</module>
       <module>module load matplotlib/2.0.0</module>
-      <module>module load intel/16.0.3</module>
       <module>module load netcdf/4.4.1.1</module>
       <module>module load nco/4.6.2</module>
       <module>module load netcdf4-python/1.2.7</module>
diff --git a/Templates/batch_cheyenne.tmpl b/Templates/batch_cheyenne.tmpl
index 57f8bef6..475412d9 100644
--- a/Templates/batch_cheyenne.tmpl
+++ b/Templates/batch_cheyenne.tmpl
@@ -5,8 +5,6 @@
 ##
 ## Averages:
 ##
-## TODO - add some hints for cheyenne queues
-##
 ## For ocean hi-resolution or atm data sets with a lot of variables, 
 ## set the netcdf_format XML variable to netcdfLarge, change the queue to
 ## either geyser (shared) or bigmem (exclusive). For geyser, set -n to 16
@@ -18,7 +16,6 @@
 ##
 ## Diagnostics:
 ##
-## TODO - add some hints for cheyenne queues
 ## NOTE - geyser and caldera aren't accessible from cheyenne yet as of 1/31/2017.
 ##
 ## For diagnostics, the queue should always be set to small, geyser or caldera
@@ -30,8 +27,6 @@
 ##
 ## Variable Time series generation:
 ##
-## TODO - add some hints for cheyenne queues
-##
 ## Load balance depends on number of history streams, 
 ## variables per stream and chunk sizes.
 ##