From fe4a1ae3423832b5010288da594c3a1aac2b4c6f Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 2 Jan 2019 16:39:13 -0700 Subject: [PATCH 1/9] Hook up expected fails functionality in run_sys_tests This is based on my cime branch, cs_status_expected_fails --- Externals.cfg | 4 +- cime_config/testdefs/ExpectedTestFails.xml | 141 ++++++++++++++++++--- python/ctsm/run_sys_tests.py | 18 ++- 3 files changed, 136 insertions(+), 27 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 42b4376bad..b1b2c17194 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,8 +29,8 @@ required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/ESMCI/cime -tag = cime5.7.5 +repo_url = https://github.com/billsacks/cime +hash = b8463d6b306809fe9f2b61b4e3856d399c16eac8 required = True [externals_description] diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 0148d01503..a3905031fc 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -1,24 +1,123 @@ - - - FAIL ERP_D_Ld10_P36x2.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_decStart RUN - FAIL ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropGs.cheyenne_gnu.clm-monthly RUN - FAIL SMS.f10_f10_musgs.I2000Clm50BgcCrop.hobart_pgi.clm-crop RUN - FAIL SMS_D.f10_f10_musgs.I2000Clm50BgcCrop.hobart_pgi.clm-crop RUN - FAIL ERS_D_Ln9_P480x3.f19_g16.I2000Clm50SpGs.cheyenne_intel.clm-waccmx_offline COMPARE_base_rest - FAIL PFS_Ld20.f09_g17.I2000Clm50BgcCrop.cheyenne_intel GENERATE - FAIL PFS_Ld20.f09_g17.I2000Clm50BgcCrop.cheyenne_intel BASELINE - - - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.hobart_nag.clm-Fates COMPARE_base_rest - FAIL ERP_Ld9.f45_f45_mg37.I2000Clm45Fates.hobart_nag.clm-FatesAllVars COMPARE_base_rest - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesLogging COMPARE_base_rest - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-Fates COMPARE_base_rest - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesNoFire COMPARE_base_rest - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesPPhys COMPARE_base_rest - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesST3 COMPARE_base_rest - FAIL ERP_Ld9.f45_f45_mg37.I2000Clm45Fates.cheyenne_intel.clm-FatesAllVars COMPARE_base_rest - FAIL ERS_Ld60.f45_f45_mg37.I2000Clm45Fates.cheyenne_gnu.clm-Fates COMPARE_base_rest - + + + + + + + FAIL + #404 + + + PEND + #404 + Because of the RUN failure, this phase is listed as PEND + + + + + + FAIL + #158 + + + + + + FAIL + #442 + + + + + + FAIL + #442 + + + + + + FAIL + #550 + + + + + + FAIL + ESMCI/cime#2917 + + + FAIL + ESMCI/cime#2917 + + + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + + + + FAIL + NGEET/fates#315 + + + diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py index 4176b70f7f..1057869a4d 100644 --- a/python/ctsm/run_sys_tests.py +++ b/python/ctsm/run_sys_tests.py @@ -10,6 +10,7 @@ from ctsm.machine_utils import get_machine_name, make_link from ctsm.machine import create_machine from ctsm.machine_defaults import MACHINE_DEFAULTS +from ctsm.path_utils import path_to_ctsm_root from CIME.test_utils import get_tests_from_xml # pylint: disable=import-error from CIME.cs_status_creator import create_cs_status # pylint: disable=import-error @@ -22,8 +23,8 @@ # For job launchers that use 'nice', the level of niceness we should use _NICE_LEVEL = 19 -# Extra arguments for the cs.status command -_CS_STATUS_EXTRA_ARGS = '--fails-only --count-performance-fails' +# Extra arguments for the cs.status.fails command +_CS_STATUS_FAILS_EXTRA_ARGS = '--fails-only --count-performance-fails' # ======================================================================== # Public functions @@ -365,20 +366,29 @@ def _make_cs_status_for_suite(testroot, testid_base): # The basic cs.status just aggregates results from all of the individual create_tests create_cs_status(test_root=testroot, test_id=testid_pattern, + extra_args=_cs_status_xfail_arg(), filename='cs.status') # cs.status.fails additionally filters the results so that only failures are shown create_cs_status(test_root=testroot, test_id=testid_pattern, - extra_args=_CS_STATUS_EXTRA_ARGS, + extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()), filename='cs.status.fails') def _make_cs_status_non_suite(testroot, testid_base): """Makes a cs.status file for a single run of create_test - not a whole test suite""" create_cs_status(test_root=testroot, test_id=testid_base, - extra_args=_CS_STATUS_EXTRA_ARGS, + extra_args=(_CS_STATUS_FAILS_EXTRA_ARGS + ' ' + _cs_status_xfail_arg()), filename='cs.status.fails') +def _cs_status_xfail_arg(): + """Returns a string giving the argument to cs_status that will point to CTSM's + expected fails xml file + """ + ctsm_root = path_to_ctsm_root() + xfail_path = os.path.join(ctsm_root, 'cime_config', 'testdefs', 'ExpectedTestFails.xml') + return "--expected-fails-file {}".format(xfail_path) + def _run_test_suite(cime_path, suite_name, suite_compilers, machine, testid_base, testroot, create_test_args, dry_run): From ba6991954f63a7a2aed53bf66495110704a88b3d Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 10 Jan 2019 12:50:25 -0700 Subject: [PATCH 2/9] Add category at the request of @ekluzek Also add some comments about the format of this file --- cime_config/testdefs/ExpectedTestFails.xml | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index a3905031fc..5d840ab3de 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -2,9 +2,42 @@ + + + + aux_clm FAIL #404 @@ -17,6 +50,7 @@ + aux_clm FAIL #158 @@ -24,6 +58,7 @@ + aux_clm FAIL #442 @@ -31,6 +66,7 @@ + aux_clm FAIL #442 @@ -38,6 +74,7 @@ + aux_clm FAIL #550 @@ -45,6 +82,7 @@ + aux_clm FAIL ESMCI/cime#2917 @@ -58,6 +96,7 @@ + fates FAIL NGEET/fates#315 @@ -65,6 +104,7 @@ + fates FAIL NGEET/fates#315 @@ -72,6 +112,7 @@ + fates FAIL NGEET/fates#315 @@ -79,6 +120,7 @@ + fates FAIL NGEET/fates#315 @@ -86,6 +128,7 @@ + fates FAIL NGEET/fates#315 @@ -93,6 +136,7 @@ + fates FAIL NGEET/fates#315 @@ -100,6 +144,7 @@ + fates FAIL NGEET/fates#315 @@ -107,6 +152,7 @@ + fates FAIL NGEET/fates#315 @@ -114,6 +160,7 @@ + fates FAIL NGEET/fates#315 From 648ac987762bcde244af736565003d9ccf87f790 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 10 Jan 2019 15:44:34 -0700 Subject: [PATCH 3/9] Point to latest version of cime branch This version includes schema validation. Note that the version number needs to be greater than 1 for cime's schema validation to kick in, due to this code in generic_xml.py: if schema is not None and self.get_version() > 1.0: self.validate_xml_file(infile, schema) --- Externals.cfg | 2 +- cime_config/testdefs/ExpectedTestFails.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index b1b2c17194..374b574e1b 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -30,7 +30,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/billsacks/cime -hash = b8463d6b306809fe9f2b61b4e3856d399c16eac8 +hash = 1e59350fe5f700a26d2ba08e0fce3cbdf1f9c6ff required = True [externals_description] diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 5d840ab3de..119e699bc7 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -1,6 +1,6 @@ - + - aux_clm FAIL #404 @@ -50,7 +43,6 @@ - aux_clm FAIL #158 @@ -58,7 +50,6 @@ - aux_clm FAIL #442 @@ -66,7 +57,6 @@ - aux_clm FAIL #442 @@ -74,7 +64,6 @@ - aux_clm FAIL #550 @@ -82,7 +71,6 @@ - aux_clm FAIL ESMCI/cime#2917 @@ -96,7 +84,6 @@ - fates FAIL NGEET/fates#315 @@ -104,7 +91,6 @@ - fates FAIL NGEET/fates#315 @@ -112,7 +98,6 @@ - fates FAIL NGEET/fates#315 @@ -120,7 +105,6 @@ - fates FAIL NGEET/fates#315 @@ -128,7 +112,6 @@ - fates FAIL NGEET/fates#315 @@ -136,7 +119,6 @@ - fates FAIL NGEET/fates#315 @@ -144,7 +126,6 @@ - fates FAIL NGEET/fates#315 @@ -152,7 +133,6 @@ - fates FAIL NGEET/fates#315 @@ -160,7 +140,6 @@ - fates FAIL NGEET/fates#315 From f70119dea917a296d4d3074a9be48d640998e3ae Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 7 Mar 2019 15:52:06 -0700 Subject: [PATCH 5/9] Point to latest cime master Among other things, this includes: - Fix for latest cheyenne system updates - Changes needed for PR #602's run_sys_tests changes --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index b1c658e633..99091ff738 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -30,7 +30,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = ctsm/ctsm1.0/cime5.7.5/n01 +hash = eb2eb704f9f2726e083162a828ad25dbe53aa09c required = True [externals_description] From fa7b01395d45f5912db1cf9b7f29f167df0477f0 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 8 Mar 2019 13:09:50 -0700 Subject: [PATCH 6/9] Point to tagged version of cime This is nearly identical to the hash I was pointing to for testing, except that this tag includes https://github.com/ESMCI/cime/pull/3030 (update CAM-SE grids). Looking through the diffs, it looks like these wouldn't at all affect any configurations we test via aux_clm. --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 99091ff738..befa477456 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -30,7 +30,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -hash = eb2eb704f9f2726e083162a828ad25dbe53aa09c +tag = cime5.7.9 required = True [externals_description] From 3356ab2efe0b25fff25148b16fbdd600576b730a Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 8 Mar 2019 13:33:01 -0700 Subject: [PATCH 7/9] SSP test: putting clone settings in a context manager block SSP tests were failing with: ERROR: Cannot modify case, read_only. Case must be opened with read_only=False and can only be modified within a context manager This change solves the problem. Regarding the settings made on `self._case` (various `self._case.set_value(...)` calls): Initially, I thought I'd also need to put those in a `with self._case` block (removing `self._case.flush()`), but looking at some examples (e.g., eri.py), it looks like these context manager blocks are mainly just used for the clone, not for `self._case`. Jim Foucar pointed out that you don't need a `with` block if the given case has already been opened higher up in the call stack, which is presumably the situation here. --- cime_config/SystemTests/ssp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/SystemTests/ssp.py b/cime_config/SystemTests/ssp.py index a2aacc9e43..fff063cbb0 100644 --- a/cime_config/SystemTests/ssp.py +++ b/cime_config/SystemTests/ssp.py @@ -55,10 +55,10 @@ def run_phase(self): logger.info(" writing restarts at end of run") logger.info(" short term archiving is on ") - clone.set_value("CLM_ACCELERATED_SPINUP", "on") - clone.set_value("MOSART_MODE", "NULL") - clone.set_value("STOP_N",stop_n1) - clone.flush() + with clone: + clone.set_value("CLM_ACCELERATED_SPINUP", "on") + clone.set_value("MOSART_MODE", "NULL") + clone.set_value("STOP_N",stop_n1) dout_sr = clone.get_value("DOUT_S_ROOT") # No history files expected, set suffix=None to avoid compare error From 765f30940cfb551228fc6dab76585df726572ee8 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 8 Mar 2019 14:46:11 -0700 Subject: [PATCH 8/9] Remove PFS fails from expected fails list: the cime update fixed this issue --- cime_config/testdefs/ExpectedTestFails.xml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 08f4527b18..ee53ca4300 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -70,17 +70,6 @@ - - - FAIL - ESMCI/cime#2917 - - - FAIL - ESMCI/cime#2917 - - - From 3558711f8274017f883094a1cf8f94be22a337d5 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 8 Mar 2019 15:08:43 -0700 Subject: [PATCH 9/9] Update ChangeLog --- doc/ChangeLog | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 146 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 45b35f3a20..223e08b82f 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,149 @@ =============================================================== +Tag name: ctsm1.0.dev030 +Originator(s): sacks (Bill Sacks) +Date: Fri Mar 8 15:08:34 MST 2019 +One-line Summary: Update CIME; hookup expected test fails + +Purpose of changes +------------------ + +Two main changes: + +(1) Update to latest cime (from ctsm/ctsm1.0/cime5.7.5/n01 to + cime5.7.9). Some of the important changes are: + - Fix for recent cheyenne system changes + - New domain file for f19_g17 (changes answers at the roundoff + level) + - New domain file for T31_g37 (changes answers by greater than + roundoff) + +(2) In system tests: Annotate cs.status output with expected failures + +Bugs fixed or introduced +------------------------ + +Issues fixed (include CTSM Issue #): +- Fixes ESCOMP/ctsm#654 + +CIME Issues fixed (include issue #): (Many) + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_0 + +[ ] clm4_5 + +Notes of particular relevance for users +--------------------------------------- + +Caveats for users (e.g., need to interpolate initial conditions): none + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): none + +Changes made to namelist defaults (e.g., changed parameter values): none + +Changes to the datasets (e.g., parameter, surface or initial files): none + +Substantial timing or memory changes: none? + The PFS test showed a significant speedup (170s in the previous tag, + 124s in this tag). However, some other tests showed either a slowdown + or not much change. So I'm not sure there is a consistent pattern of + speedup in this tag. + +Notes of particular relevance for developers: (including Code reviews and testing) +--------------------------------------------- +NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): none + +Changes to tests or testing: none + +Code reviewed by: Erik Kluzek + + +CTSM testing: + + [PASS means all tests PASS and OK means tests PASS other than expected fails.] + + build-namelist tests: + + cheyenne - not run + + tools-tests (test/tools): + + cheyenne - not run + + PTCLM testing (tools/shared/PTCLM/test): + + cheyenne - not run + + regular tests (aux_clm): + + cheyenne ---- ok + hobart ------ ok + + ok means tests pass, some baselines failed as noted below + + Most testing done with cime at eb2eb704f - excluded PR #3030, which + updated support for CAM-SE grids (but the changes in this PR were + very limited). Ran SSP tests with the final version of cime (so I'd + have a few tests covering the latest cime). + +CTSM tag used for the baseline comparisons: ctsm1.0.dev029 + + +Answer changes +-------------- + +Changes answers relative to baseline: YES + + If a tag changes answers relative to baseline comparison the + following should be filled in (otherwise remove this section): + + Summarize any changes to answers, i.e., + - what code configurations: f19_g17, T31_g37 + - what platforms/compilers: all + - nature of change (roundoff; larger than roundoff/same climate; new climate): + + - f19_g17: roundoff-level, presumably due to new domain file (some + diffs grew to greater than roundoff-level: some methane + variables in a long test - + SMS_Lm13.f19_g17.I2000Clm50BgcCrop.cheyenne_intel.clm-cropMonthOutput; + and many variables in test with flooding - + ERP_P180x2_D.f19_g17.I2000Clm50SpRtmFl.cheyenne_intel.clm-default) + + - T31_g37: greater than roundoff-level, presumably due to new + domain file from cime commit 4fcf592ee + + + If bitwise differences were observed, how did you show they were no worse + than roundoff? N/A + + If this tag changes climate describe the run(s) done to evaluate the new + climate (put details of the simulations in the experiment database) + - casename: N/A + + URL for LMWG diagnostics output used to validate new climate: N/A + + +Detailed list of changes +------------------------ + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): +- cime: ctsm/ctsm1.0/cime5.7.5/n01 -> cime5.7.9 + +Pull Requests that document the changes (include PR ids): +- https://github.com/ESCOMP/ctsm/pull/602 + +=============================================================== +=============================================================== Tag name: ctsm1.0.dev029 Originator(s): slevis (Samuel Levis, SLevis Consulting LLC, 303-665-1310) Date: Tue Feb 26 23:42:39 MST 2019 diff --git a/doc/ChangeSum b/doc/ChangeSum index 210ec1adfa..3f5532b335 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm1.0.dev030 sacks 03/08/2019 Update CIME; hookup expected test fails ctsm1.0.dev029 slevis 02/26/2019 Collapse landunits to the N most dominant ctsm1.0.dev028 sacks 02/26/2019 Interpolate out-of-the-box initial conditions and remove expensive tests ctsm1.0.dev027 negins 02/19/2019 non-constant time init for soil hydrology types are moved to SoilHydrologyType.F90