Merge tag 'ctsm1.0.dev030' into dynglacier_smaller_fluxes

Update CIME; hookup expected test fails Two main changes: (1) Update to latest cime (from ctsm/ctsm1.0/cime5.7.5/n01 to cime5.7.9). Some of the important changes are: - Fix for recent cheyenne system changes - New domain file for f19_g17 (changes answers at the roundoff level) - New domain file for T31_g37 (changes answers by greater than roundoff) (2) In system tests: Annotate cs.status output with expected failures Fixes ESCOMP#654
billsacks · Mar 12, 2019 · 4016cd7 · 4016cd7
2 parents a8263c1 + b26c12f
commit 4016cd7
Show file tree

Hide file tree

Showing 6 changed files with 299 additions and 30 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -30,7 +30,7 @@ required = True
 local_path = cime
 protocol = git
 repo_url = https://github.com/ESMCI/cime
-tag = ctsm/ctsm1.0/cime5.7.5/n01
+tag = cime5.7.9
 required = True
 
 [externals_description]

diff --git a/cime_config/SystemTests/ssp.py b/cime_config/SystemTests/ssp.py
@@ -55,10 +55,10 @@ def run_phase(self):
         logger.info("  writing restarts at end of run")
         logger.info("  short term archiving is on ")
 
-        clone.set_value("CLM_ACCELERATED_SPINUP", "on")
-        clone.set_value("MOSART_MODE", "NULL")
-        clone.set_value("STOP_N",stop_n1)
-        clone.flush()
+        with clone:
+            clone.set_value("CLM_ACCELERATED_SPINUP", "on")
+            clone.set_value("MOSART_MODE", "NULL")
+            clone.set_value("STOP_N",stop_n1)
 
         dout_sr = clone.get_value("DOUT_S_ROOT")
         # No history files expected, set suffix=None to avoid compare error

diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml
@@ -1,24 +1,138 @@
 <?xml version= "1.0"?>
 
-<expectedFails>
-  <category name="aux_clm">
-    <entry issue="#404"            >FAIL ERP_D_Ld10_P36x2.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_decStart RUN</entry>
-    <entry issue="#158"            >FAIL ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropGs.cheyenne_gnu.clm-monthly RUN</entry>
-    <entry issue="#442"            >FAIL SMS.f10_f10_musgs.I2000Clm50BgcCrop.hobart_pgi.clm-crop RUN</entry>
-    <entry issue="#442"            >FAIL SMS_D.f10_f10_musgs.I2000Clm50BgcCrop.hobart_pgi.clm-crop RUN</entry>
-    <entry issue="#550"            >FAIL ERS_D_Ln9_P480x3.f19_g16.I2000Clm50SpGs.cheyenne_intel.clm-waccmx_offline COMPARE_base_rest</entry>
-    <entry issue="ESMCI/cime#2917" >FAIL PFS_Ld20.f09_g17.I2000Clm50BgcCrop.cheyenne_intel GENERATE</entry>
-    <entry issue="ESMCI/cime#2917" >FAIL PFS_Ld20.f09_g17.I2000Clm50BgcCrop.cheyenne_intel BASELINE</entry>
-  </category>
-  <category name="fates">
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.hobart_nag.clm-Fates COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERP_Ld9.f45_f45_mg37.I2000Clm45Fates.hobart_nag.clm-FatesAllVars COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesLogging COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-Fates COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesNoFire COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesPPhys COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesST3 COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERP_Ld9.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesAllVars COMPARE_base_rest</entry>
-    <entry issue="NGEET/fates#315">FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_gnu.clm-Fates COMPARE_base_rest</entry>
-  </category>
+<expectedFails version="1.1">
+
+  <!-- Notes about the format of this file:
+
+       The required elements for a given failure are just:
+
+         <test name="...">
+           <phase name="...">
+             <status>...</status>
+           </phase>
+         </test>
+
+       There can be multiple phase blocks in a given test block.
+
+       In addition, a number of optional elements are allowed, which
+       currently are just for human consumption (not parsed by any
+       scripts):
+
+       - A phase block can contain an "issue" element, which gives the
+       issue number associated with this failure. (#123 refers to issue
+       #123 in the ESCOMP/ctsm repository. Issues in other repositories
+       should be specified as ORG/repo#123 - e.g., ESMCI/cime#123.)
+
+       - A phase block can contain a "comment" element, which gives any
+       sort of comment you desire.
+  -->
+
+
+  <!-- aux_clm test suite failures -->
+
+  <test name="ERP_D_Ld10_P36x2.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_decStart">
+    <phase name="RUN">
+      <status>FAIL</status>
+      <issue>#404</issue>
+    </phase>
+    <phase name="COMPARE_base_rest">
+      <status>PEND</status>
+      <issue>#404</issue>
+      <comment>Because of the RUN failure, this phase is listed as PEND</comment>
+    </phase>
+  </test>
+
+  <test name="ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropGs.cheyenne_gnu.clm-monthly">
+    <phase name="RUN">
+      <status>FAIL</status>
+      <issue>#158</issue>
+    </phase>
+  </test>
+
+  <test name="SMS.f10_f10_musgs.I2000Clm50BgcCrop.hobart_pgi.clm-crop">
+    <phase name="RUN">
+      <status>FAIL</status>
+      <issue>#442</issue>
+    </phase>
+  </test>
+
+  <test name="SMS_D.f10_f10_musgs.I2000Clm50BgcCrop.hobart_pgi.clm-crop">
+    <phase name="RUN">
+      <status>FAIL</status>
+      <issue>#442</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_D_Ln9_P480x3.f19_g16.I2000Clm50SpGs.cheyenne_intel.clm-waccmx_offline">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>#550</issue>
+    </phase>
+  </test>
+
+  <!-- fates test suite failures -->
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.hobart_nag.clm-Fates">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERP_Ld9.f45_f45_mg37.I2000Clm45Fates.hobart_nag.clm-FatesAllVars">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesLogging">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-Fates">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesNoFire">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesPPhys">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesST3">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERP_Ld9.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesAllVars">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
+  <test name="ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_gnu.clm-Fates">
+    <phase name="COMPARE_base_rest">
+      <status>FAIL</status>
+      <issue>NGEET/fates#315</issue>
+    </phase>
+  </test>
+
 </expectedFails> 
diff --git a/doc/ChangeLog b/doc/ChangeLog
@@ -1,4 +1,149 @@
 ===============================================================
+Tag name:  ctsm1.0.dev030
+Originator(s):  sacks (Bill Sacks)
+Date: Fri Mar  8 15:08:34 MST 2019
+One-line Summary: Update CIME; hookup expected test fails
+
+Purpose of changes
+------------------
+
+Two main changes:
+
+(1) Update to latest cime (from ctsm/ctsm1.0/cime5.7.5/n01 to
+    cime5.7.9). Some of the important changes are:
+    - Fix for recent cheyenne system changes
+    - New domain file for f19_g17 (changes answers at the roundoff
+      level)
+    - New domain file for T31_g37 (changes answers by greater than
+      roundoff)
+
+(2) In system tests: Annotate cs.status output with expected failures
+
+Bugs fixed or introduced
+------------------------
+
+Issues fixed (include CTSM Issue #):
+- Fixes ESCOMP/ctsm#654
+
+CIME Issues fixed (include issue #): (Many)
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_0
+
+[ ] clm4_5
+
+Notes of particular relevance for users
+---------------------------------------
+
+Caveats for users (e.g., need to interpolate initial conditions): none
+
+Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): none
+
+Changes made to namelist defaults (e.g., changed parameter values): none
+
+Changes to the datasets (e.g., parameter, surface or initial files): none
+
+Substantial timing or memory changes: none?
+   The PFS test showed a significant speedup (170s in the previous tag,
+   124s in this tag). However, some other tests showed either a slowdown
+   or not much change. So I'm not sure there is a consistent pattern of
+   speedup in this tag.
+
+Notes of particular relevance for developers: (including Code reviews and testing)
+---------------------------------------------
+NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide
+
+Caveats for developers (e.g., code that is duplicated that requires double maintenance): none
+
+Changes to tests or testing: none
+
+Code reviewed by: Erik Kluzek
+
+
+CTSM testing:
+
+ [PASS means all tests PASS and OK means tests PASS other than expected fails.]
+
+  build-namelist tests:
+
+    cheyenne - not run
+
+  tools-tests (test/tools):
+
+    cheyenne - not run
+
+  PTCLM testing (tools/shared/PTCLM/test):
+
+     cheyenne - not run
+
+  regular tests (aux_clm):
+
+    cheyenne ---- ok
+    hobart ------ ok
+
+    ok means tests pass, some baselines failed as noted below
+
+    Most testing done with cime at eb2eb704f - excluded PR #3030, which
+    updated support for CAM-SE grids (but the changes in this PR were
+    very limited). Ran SSP tests with the final version of cime (so I'd
+    have a few tests covering the latest cime).
+
+CTSM tag used for the baseline comparisons: ctsm1.0.dev029
+
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: YES
+
+  If a tag changes answers relative to baseline comparison the
+  following should be filled in (otherwise remove this section):
+
+  Summarize any changes to answers, i.e.,
+    - what code configurations: f19_g17, T31_g37
+    - what platforms/compilers: all
+    - nature of change (roundoff; larger than roundoff/same climate; new climate): 
+
+      - f19_g17: roundoff-level, presumably due to new domain file (some
+        diffs grew to greater than roundoff-level: some methane
+        variables in a long test -
+        SMS_Lm13.f19_g17.I2000Clm50BgcCrop.cheyenne_intel.clm-cropMonthOutput;
+        and many variables in test with flooding -
+        ERP_P180x2_D.f19_g17.I2000Clm50SpRtmFl.cheyenne_intel.clm-default)
+
+      - T31_g37: greater than roundoff-level, presumably due to new
+        domain file from cime commit 4fcf592ee
+
+
+   If bitwise differences were observed, how did you show they were no worse
+   than roundoff? N/A
+
+   If this tag changes climate describe the run(s) done to evaluate the new
+   climate (put details of the simulations in the experiment database)
+       - casename: N/A
+
+   URL for LMWG diagnostics output used to validate new climate: N/A
+
+
+Detailed list of changes
+------------------------
+
+List any externals directories updated (cime, rtm, mosart, cism, fates, etc.):
+- cime: ctsm/ctsm1.0/cime5.7.5/n01 -> cime5.7.9
+
+Pull Requests that document the changes (include PR ids):
+- https://github.com/ESCOMP/ctsm/pull/602
+
+===============================================================
+===============================================================
 Tag name:  ctsm1.0.dev029
 Originator(s):  slevis (Samuel Levis, SLevis Consulting LLC, 303-665-1310)
 Date: Tue Feb 26 23:42:39 MST 2019

diff --git a/doc/ChangeSum b/doc/ChangeSum
@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+  ctsm1.0.dev030    sacks 03/08/2019 Update CIME; hookup expected test fails
   ctsm1.0.dev029   slevis 02/26/2019 Collapse landunits to the N most dominant
   ctsm1.0.dev028    sacks 02/26/2019 Interpolate out-of-the-box initial conditions and remove expensive tests
   ctsm1.0.dev027   negins 02/19/2019 non-constant time init for soil hydrology types are moved to SoilHydrologyType.F90

diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py
@@ -25,8 +25,8 @@
 # For job launchers that use 'nice', the level of niceness we should use
 _NICE_LEVEL = 19
 
-# Extra arguments for the cs.status command
-_CS_STATUS_EXTRA_ARGS = '--fails-only --count-performance-fails'
+# Extra arguments for the cs.status.fails command
+_CS_STATUS_FAILS_EXTRA_ARGS = '--fails-only --count-performance-fails'
 
 # ========================================================================
 # Public functions
@@ -424,20 +424,29 @@ def _make_cs_status_for_suite(testroot, testid_base):
     # The basic cs.status just aggregates results from all of the individual create_tests
     create_cs_status(test_root=testroot,
                      test_id=testid_pattern,
+                     extra_args=_cs_status_xfail_arg(),
                      filename='cs.status')
     # cs.status.fails additionally filters the results so that only failures are shown
     create_cs_status(test_root=testroot,
                      test_id=testid_pattern,
-                     extra_args=_CS_STATUS_EXTRA_ARGS,
+                     extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()),
                      filename='cs.status.fails')
 
 def _make_cs_status_non_suite(testroot, testid_base):
     """Makes a cs.status file for a single run of create_test - not a whole test suite"""
     create_cs_status(test_root=testroot,
                      test_id=testid_base,
-                     extra_args=_CS_STATUS_EXTRA_ARGS,
+                     extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()),
                      filename='cs.status.fails')
 
+def _cs_status_xfail_arg():
+    """Returns a string giving the argument to cs_status that will point to CTSM's
+    expected fails xml file
+    """
+    ctsm_root = path_to_ctsm_root()
+    xfail_path = os.path.join(ctsm_root, 'cime_config', 'testdefs', 'ExpectedTestFails.xml')
+    return "--expected-fails-file {}".format(xfail_path)
+
 def _run_test_suite(cime_path, suite_name, suite_compilers,
                     machine, testid_base, testroot, create_test_args,
                     dry_run):