From c7110000102dedc386cf49b5a56f172aaf4e55d2 Mon Sep 17 00:00:00 2001
From: Azamat Mametjanov <azamat@mcs.anl.gov>
Date: Thu, 7 Sep 2017 23:22:32 +0000
Subject: [PATCH] Add mach-specific throughput checking to test machines

---
 config/acme/machines/config_machines.xml            | 11 +++++++++++
 scripts/lib/CIME/SystemTests/system_tests_common.py |  8 ++++++--
 scripts/lib/CIME/test_scheduler.py                  |  1 +
 src/drivers/mct/cime_config/config_component.xml    |  8 ++++++++
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml
index be8c8056130..e0dbd52df00 100644
--- a/config/acme/machines/config_machines.xml
+++ b/config/acme/machines/config_machines.xml
@@ -79,6 +79,7 @@
   <BASELINE_ROOT>/project/projectdirs/acme/baselines</BASELINE_ROOT>
   <CCSM_CPRNC>/project/projectdirs/acme/tools/cprnc.edison/cprnc</CCSM_CPRNC>
   <SAVE_TIMING_DIR>/project/projectdirs/$PROJECT</SAVE_TIMING_DIR>
+  <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
   <mpirun mpilib="default">
     <executable>srun</executable>
     <arguments>
@@ -208,6 +209,7 @@
   <PROJECT>acme</PROJECT>
   <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
   <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
+  <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
   <mpirun mpilib="default">
     <executable>srun</executable>
     <arguments>
@@ -348,6 +350,7 @@
   <PROJECT>acme</PROJECT>
   <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
   <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
+  <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
   <mpirun mpilib="default">
     <executable>srun</executable>
     <arguments>
@@ -558,6 +561,7 @@
     <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
     <PES_PER_NODE>64</PES_PER_NODE>
     <BATCH_SYSTEM>none</BATCH_SYSTEM>
+    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
     <mpirun mpilib="default">
       <executable>mpirun</executable>
       <arguments>
@@ -742,6 +746,7 @@
   <PIO_BUFFER_SIZE_LIMIT>1</PIO_BUFFER_SIZE_LIMIT>
   <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
   <PROJECT>fy150001</PROJECT>
+  <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
 
   <mpirun mpilib="default">
     <executable>mpiexec</executable>
@@ -810,6 +815,7 @@
   <PIO_BUFFER_SIZE_LIMIT>1</PIO_BUFFER_SIZE_LIMIT>
   <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
   <PROJECT>fy150001</PROJECT>
+  <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
 
   <mpirun mpilib="default">
     <executable>mpiexec</executable>
@@ -943,6 +949,7 @@
 	 <PES_PER_NODE>16</PES_PER_NODE>
          <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
 	 <PROJECT>ACME</PROJECT>
+         <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
          <mpirun mpilib="mvapich">
            <executable>mpiexec</executable>
            <arguments>
@@ -1150,6 +1157,7 @@
     <PROJECT>acme</PROJECT>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
+    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
     <mpirun mpilib="default"> 
       <executable>mpirun</executable>
       <arguments>
@@ -1227,6 +1235,7 @@
          <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
          <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
          <PROJECT>ClimateEnergy_2</PROJECT>
+         <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
          <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
          <mpirun mpilib="default">
            <executable>/usr/bin/runjob</executable>
@@ -1444,6 +1453,7 @@
          <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
          <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
          <PROJECT>ClimateEnergy_2</PROJECT>
+         <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
          <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
          <mpirun mpilib="default">
            <executable>/usr/bin/runjob</executable>
@@ -1990,6 +2000,7 @@
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <PROJECT>cli115</PROJECT>
     <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
+    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
     <mpirun mpilib="default">
       <executable args="default">aprun</executable>
       <arguments>
diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py
index a690386d474..0f7433cfec5 100644
--- a/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -387,10 +387,14 @@ def _compare_baseline(self):
             #comparing ypd so bigger is better
             if baseline is not None and current is not None:
                 diff = (baseline - current)/baseline
-                if(diff < 0.25):
+                tolerance = self._case.get_value("TEST_TPUT_TOLERANCE")
+                if tolerance is None:
+                    tolerance = 0.25
+                expect(tolerance > 0.0, "Bad value for throughput tolerance in test")
+                if diff < tolerance:
                     self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS)
                 else:
-                    comment = "Error: Computation time increase > 25% from baseline"
+                    comment = "Error: Computation time increase > %f pct from baseline" % tolerance*100
                     self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment)
                     append_testlog(comment)
 
diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py
index 4c6d289796c..51785d3a26e 100644
--- a/scripts/lib/CIME/test_scheduler.py
+++ b/scripts/lib/CIME/test_scheduler.py
@@ -467,6 +467,7 @@ def _xml_phase(self, test):
         envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None)
         envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None)
         envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
+        envtest.set_value("TEST_TPUT_TOLERANCE", self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False))
 
         # Add the test instructions from config_test to env_test in the case
         config_test = Tests()
diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml
index 449adaaa303..661d69a0943 100644
--- a/src/drivers/mct/cime_config/config_component.xml
+++ b/src/drivers/mct/cime_config/config_component.xml
@@ -2367,6 +2367,14 @@
     <desc>Expected relative memory usage growth for test</desc>
   </entry>
 
+  <entry id="TEST_TPUT_TOLERANCE">
+    <type>real</type>
+    <default_value>0.25</default_value>
+    <group>test</group>
+    <file>env_test.xml</file>
+    <desc>Expected throughput deviation</desc>
+  </entry>
+
   <entry id="GENERATE_BASELINE">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>