Merge pull request #392 from ESMCI/jgfouca/guard_against_test_obj_ini…

…t_throw

Exceptions in SystemTest constructors should leave TestStatus in decent state

Also,
1) Added regression tests to test the above
2) case_setup should not put stuff in TestStatus unless it's a test case
3) Rename some acme_* things to cime_*

Test suite: scripts_regresssion_test
Test baseline: 
Test namelist changes: 
Test status: bit for bit

Fixes #390

User interface changes?: Minor

Code review: @billsacks @jedwards4b

scripts/Testing/Testcases/config_tests.xml

@@ -217,6 +217,16 @@ LAR    long term archive test
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
+  <test NAME="TESTBUILDFAILEXC">
+    <DESC>For testing infra only. Insta-fail build step by failing to init.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <CCSM_TCOST>0</CCSM_TCOST>
+    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_N>11</STOP_N>
+    <CHECK_TIMING>FALSE</CHECK_TIMING>
+    <DOUT_S>FALSE</DOUT_S>
+  </test>
+
   <test NAME="TESTRUNFAIL">
     <DESC>For testing infra only. Insta-fail run step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
@@ -227,6 +237,16 @@ LAR    long term archive test
     <DOUT_S>FALSE</DOUT_S>
   </test>
 
+  <test NAME="TESTRUNFAILEXC">
+    <DESC>For testing infra only. Insta-fail run step via exception.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <CCSM_TCOST>0</CCSM_TCOST>
+    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_N>11</STOP_N>
+    <CHECK_TIMING>FALSE</CHECK_TIMING>
+    <DOUT_S>FALSE</DOUT_S>
+  </test>
+
   <test NAME="TESTRUNPASS">
     <DESC>For testing infra only. Insta-pass run step.</DESC>
     <INFO_DBUG>1</INFO_DBUG>

scripts/Tools/case.build

@@ -11,6 +11,7 @@ from CIME.case import Case
 from CIME.utils           import expect, append_status, find_system_test
 from CIME.XML.files       import Files
 from CIME.XML.component   import Component
+from CIME.test_status     import *
 
 ###############################################################################
 def parse_command_line(args, description):
@@ -85,7 +86,17 @@ def _main_func(description):
         elif(testname is not None):
             logging.warn("Building test for %s in directory %s" %
                          (testname, caseroot))
-            test = find_system_test(testname, case)(case)
+            try:
+                # The following line can throw exceptions if the testname is
+                # not found or the test constructor throws. We need to be
+                # sure to leave TestStatus in the appropriate state if that
+                # happens.
+                test = find_system_test(testname, case)(case)
+            except:
+                phase_to_fail = MODEL_BUILD_PHASE if model_only else SHAREDLIB_BUILD_PHASE
+                with TestStatus(test_dir=caseroot) as ts:
+                    ts.set_status(phase_to_fail, TEST_FAIL_STATUS, comments="failed to initialize")
+                raise
 
             append_status("case.testbuild starting ",
                           caseroot=caseroot,sfile="CaseStatus")

utils/python/CIME/SystemTests/system_tests_common.py

@@ -491,11 +491,22 @@ def build_phase(self, sharedlib_only=False, model_only=False):
         FakeTest.build_phase(self,
                              sharedlib_only=sharedlib_only, model_only=model_only)
 
+class TESTRUNFAILEXC(TESTRUNPASS):
+
+    def run_phase(self):
+        raise RuntimeError("Exception from run_phase")
+
 class TESTBUILDFAIL(FakeTest):
 
     def build_phase(self, sharedlib_only=False, model_only=False):
         if (not sharedlib_only):
-            expect(False, "ERROR: Intentional fail for testing infrastructure")
+            expect(False, "Intentional fail for testing infrastructure")
+
+class TESTBUILDFAILEXC(FakeTest):
+
+    def __init__(self, case):
+        FakeTest.__init__(self, case)
+        raise RuntimeError("Exception from init")
 
 class TESTRUNSLOWPASS(FakeTest):
 

utils/python/CIME/case_setup.py

@@ -295,12 +295,15 @@ def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, r
 def case_setup(case, clean=False, test_mode=False, reset=False):
 ###############################################################################
     caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID")
-    test_name = casebaseid if casebaseid is not None else case.get_value("CASE")
-    with TestStatus(test_dir=caseroot, test_name=test_name) as ts:
-        try:
-            _case_setup_impl(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset)
-        except:
-            ts.set_status(SETUP_PHASE, TEST_FAIL_STATUS)
-            raise
-        else:
-            ts.set_status(SETUP_PHASE, TEST_PASS_STATUS)
+    if case.get_value("TEST"):
+        test_name = casebaseid if casebaseid is not None else case.get_value("CASE")
+        with TestStatus(test_dir=caseroot, test_name=test_name) as ts:
+            try:
+                _case_setup_impl(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset)
+            except:
+                ts.set_status(SETUP_PHASE, TEST_FAIL_STATUS)
+                raise
+            else:
+                ts.set_status(SETUP_PHASE, TEST_PASS_STATUS)
+    else:
+        _case_setup_impl(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset)

utils/python/CIME/case_test.py

@@ -13,7 +13,18 @@ def case_test(case, testname=None):
     expect(testname is not None, "testname argument not resolved")
     logging.warn("Running test for %s" % testname)
 
-    test = find_system_test(testname, case)(case)
+    try:
+        # The following line can throw exceptions if the testname is
+        # not found or the test constructor throws. We need to be
+        # sure to leave TestStatus in the appropriate state if that
+        # happens.
+        test = find_system_test(testname, case)(case)
+    except:
+        caseroot = case.get_value("CASEROOT")
+        with TestStatus(test_dir=caseroot) as ts:
+            ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize")
+        raise
+
     success = test.run()
 
     return success

utils/python/tests/scripts_regression_tests.py

@@ -425,7 +425,7 @@ def simple_test(self, expect_works, extra_args):
     ###########################################################################
         if NO_BATCH:
             extra_args += " --no-batch"
-        cmd = "%s/create_test acme_test_only_pass %s" % (SCRIPT_DIR, extra_args)
+        cmd = "%s/create_test cime_test_only_pass %s" % (SCRIPT_DIR, extra_args)
         stat, output, errput = run_cmd(cmd)
         if (expect_works):
             self.assertEqual(stat, 0, msg="COMMAND '%s' SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (cmd, output, errput, stat))
@@ -479,8 +479,11 @@ class E_TestTestScheduler(TestCreateTestCommon):
     def test_a_phases(self):
     ###########################################################################
         # exclude the MEMLEAK tests here.
-        tests = update_acme_tests.get_full_test_names(["acme_test_only",
-                                                       "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X", "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X"],
+        tests = update_acme_tests.get_full_test_names(["cime_test_only",
+                                                       "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X",
+                                                       "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X",
+                                                       "^TESTBUILDFAILEXC.f19_g16_rx1.A",
+                                                       "^TESTRUNFAILEXC_Mmpi-serial.f19_g16_rx1.A"],
                                                       self._machine, self._compiler)
         self.assertEqual(len(tests), 3)
         ct = TestScheduler(tests)
@@ -551,15 +554,17 @@ def test_a_phases(self):
     ###########################################################################
     def test_b_full(self):
     ###########################################################################
-        tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler)
+        tests = update_acme_tests.get_full_test_names(["cime_test_only"], self._machine, self._compiler)
         test_id="%s-%s" % (self._baseline_name, CIME.utils.get_utc_timestamp())
         ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH)
 
-        build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0]
-        run_fail_test   = [item for item in tests if "TESTRUNFAIL" in item][0]
-        pass_test       = [item for item in tests if "TESTRUNPASS" in item][0]
-        mem_fail_test   = [item for item in tests if "TESTMEMLEAKFAIL" in item][0]
-        mem_pass_test   = [item for item in tests if "TESTMEMLEAKPASS" in item][0]
+        build_fail_test     = [item for item in tests if "TESTBUILDFAIL." in item][0]
+        build_fail_exc_test = [item for item in tests if "TESTBUILDFAILEXC" in item][0]
+        run_fail_test       = [item for item in tests if "TESTRUNFAIL_" in item][0]
+        run_fail_exc_test   = [item for item in tests if "TESTRUNFAILEXC" in item][0]
+        pass_test           = [item for item in tests if "TESTRUNPASS" in item][0]
+        mem_fail_test       = [item for item in tests if "TESTMEMLEAKFAIL" in item][0]
+        mem_pass_test       = [item for item in tests if "TESTMEMLEAKPASS" in item][0]
 
         log_lvl = logging.getLogger().getEffectiveLevel()
         logging.disable(logging.CRITICAL)
@@ -584,8 +589,16 @@ def test_b_full(self):
                 self.assertEqual(ts.get_status(CIME.test_scheduler.MODEL_BUILD_PHASE), TEST_FAIL_STATUS)
                 self.assertTrue("Intentional fail for testing infrastructure" in open(log_file, "r").read(),
                                 "Broken test did not report build error")
+            elif (test_name == build_fail_exc_test):
+                self.assertEqual(ts.get_status(CIME.test_scheduler.SHAREDLIB_BUILD_PHASE), TEST_FAIL_STATUS)
+                self.assertTrue("Exception from init" in open(log_file, "r").read(),
+                                "Broken test did not report build error")
             elif (test_name == run_fail_test):
                 self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_FAIL_STATUS)
+            elif (test_name == run_fail_exc_test):
+                self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_FAIL_STATUS)
+                self.assertTrue("Exception from run_phase" in open(log_file, "r").read(),
+                                "Broken test did not report build error")
             elif (test_name == mem_fail_test):
                 self.assertEqual(ts.get_status(MEMLEAK_PHASE), TEST_FAIL_STATUS)
                 self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS)
@@ -644,7 +657,7 @@ def assert_num_leftovers(self, test_id=None):
         # the testroot (bld/run dump area) and jenkins root
         if (test_id is None):
             test_id = self._baseline_name
-        num_tests_in_tiny = len(update_acme_tests.get_test_suite("acme_test_only_pass"))
+        num_tests_in_tiny = len(update_acme_tests.get_test_suite("cime_test_only_pass"))
 
         jenkins_dirs = glob.glob("%s/*%s*/" % (self._jenkins_root, test_id)) # case dirs
         # scratch_dirs = glob.glob("%s/*%s*/" % (self._testroot, test_id)) # blr/run dirs
@@ -665,19 +678,19 @@ def test_jenkins_generic_job(self):
 
         # Generate fresh baselines so that this test is not impacted by
         # unresolved diffs
-        self.simple_test(True, "-t acme_test_only_pass -g -b %s" % self._baseline_name)
+        self.simple_test(True, "-t cime_test_only_pass -g -b %s" % self._baseline_name)
         self.assert_num_leftovers()
 
         build_name = "jenkins_generic_job_pass_%s" % CIME.utils.get_utc_timestamp()
-        self.simple_test(True, "-t acme_test_only_pass -b %s" % self._baseline_name, build_name=build_name)
+        self.simple_test(True, "-t cime_test_only_pass -b %s" % self._baseline_name, build_name=build_name)
         self.assert_num_leftovers() # jenkins_generic_job should have automatically cleaned up leftovers from prior run
         assert_dashboard_has_build(self, build_name)
 
     ###########################################################################
     def test_jenkins_generic_job_kill(self):
     ###########################################################################
         build_name = "jenkins_generic_job_kill_%s" % CIME.utils.get_utc_timestamp()
-        run_thread = threading.Thread(target=self.threaded_test, args=(False, " -t acme_test_only_slow_pass -b master --baseline-compare=no", build_name))
+        run_thread = threading.Thread(target=self.threaded_test, args=(False, " -t cime_test_only_slow_pass -b master --baseline-compare=no", build_name))
         run_thread.daemon = True
         run_thread.start()
 
@@ -768,7 +781,7 @@ def test_update_acme_tests(self):
     ###########################################################################
         # Add some testable stuff to acme tests
         pass
-        # update_acme_tests._TEST_SUITES["acme_tiny"] = \
+        # update_acme_tests._TEST_SUITES["cime_tiny"] = \
         #     (None, (("ERS.f19_g16_rx1.A", "jgftestmodtest/test_mod"),
         #             ("NCK.f19_g16_rx1.A", "jgftestmodtest/test_mod"))
         #      )
@@ -790,15 +803,15 @@ def test_update_acme_tests_test_mods(self):
         # not_my_machine = "%s_jgftest" % machine
 
         # # Add some testable stuff to acme tests
-        # update_acme_tests._TEST_SUITES["acme_tiny"] = \
+        # update_acme_tests._TEST_SUITES["cime_tiny"] = \
         #     (None, (("ERS.f19_g16_rx1.A", "test_mod"),
         #             ("ERS.f19_g16_rx1.B", "test_mod", machine),
         #             ("ERS.f19_g16_rx1.C", "test_mod", (machine, not_my_machine)),
         #             ("ERS.f19_g16_rx1.D", "test_mod", not_my_machine),
         #             "ERS.f19_g16_rx1.E")
         #      )
 
-        # tests = update_acme_tests.get_test_suite("acme_tiny", compiler="gnu")
+        # tests = update_acme_tests.get_test_suite("cime_tiny", compiler="gnu")
 
         # self.assertEqual(5, len(tests))
         # self.assertTrue("ERS.f19_g16_rx1.A.melvin_gnu.test_mod" in tests)
@@ -861,7 +874,7 @@ class TestCimeCase(TestCreateTestCommon):
     ###########################################################################
     def test_cime_case(self):
     ###########################################################################
-        stat, output, errput = run_cmd("%s/create_test acme_test_only -t %s --no-build" % (SCRIPT_DIR, self._baseline_name))
+        stat, output, errput = run_cmd("%s/create_test cime_test_only -t %s --no-build" % (SCRIPT_DIR, self._baseline_name))
         self.assertEqual(stat, 0,
                          msg="COMMAND SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (output, errput, stat))
 

utils/python/update_acme_tests.py

@@ -10,31 +10,47 @@
 #   If testmods are needed, a 2-ple must be provided  (test, mods)
 #   If you want to restrict the test mods to certain machines, than a 3-ple is needed (test, mods, [machines])
 _TEST_SUITES = {
-    "acme_tiny" : (None,
+    "cime_tiny" : (None,
                    ("ERS.f19_g16_rx1.A",
                     "NCK.f19_g16_rx1.A")
                    ),
 
-    "acme_test_only_pass" : (None,
+    "cime_test_only_pass" : (None,
                    ("TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A",
                     "TESTRUNPASS_Mmpi-serial.ne30_g16_rx1.A",
                     "TESTRUNPASS_Mmpi-serial.f45_g37_rx1.A")
                    ),
 
-    "acme_test_only_slow_pass" : (None,
+    "cime_test_only_slow_pass" : (None,
                    ("TESTRUNSLOWPASS_Mmpi-serial.f19_g16_rx1.A",
                     "TESTRUNSLOWPASS_Mmpi-serial.ne30_g16_rx1.A",
                     "TESTRUNSLOWPASS_Mmpi-serial.f45_g37_rx1.A")
                    ),
 
-    "acme_test_only" : (None,
+    "cime_test_only" : (None,
                    ("TESTBUILDFAIL.f19_g16_rx1.A",
+                    "TESTBUILDFAILEXC.f19_g16_rx1.A",
                     "TESTRUNFAIL_Mmpi-serial.f19_g16_rx1.A",
+                    "TESTRUNFAILEXC_Mmpi-serial.f19_g16_rx1.A",
                     "TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A",
                     "TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X",
                     "TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X")
                    ),
 
+    "cime_developer" : (None,
+                            ("NCK_Ld3.f45_g37_rx1.A",
+                             "ERI.f45_g37.X",
+                             "SEQ_Ln9.f19_g16_rx1.A",
+                             "ERS_Ld3.ne30_g16_rx1.A",
+                             "ERS_N2_Ld3.f19_g16_rx1.A",
+                             "ERR_Ld3.f45_g37_rx1.A",
+                             "SMS_D_Ln9_Mmpi-serial.f19_g16_rx1.A")
+                            ),
+
+    #
+    # ACME tests below
+    #
+
     "acme_runoff_developer" : (None,
                              ("SMS.f19_f19.IM1850CLM45CN",
                               "SMS.f19_f19.IMCLM45")
@@ -63,16 +79,6 @@
                              "SMS_D.f19_g16.FC5ATMMODCOSP")
                             ),
 
-    "cime_developer" : (None,
-                            ("NCK_Ld3.f45_g37_rx1.A",
-                             "ERI.f45_g37.X",
-                             "SEQ_Ln9.f19_g16_rx1.A",
-                             "ERS_Ld3.ne30_g16_rx1.A",
-                             "ERS_N2_Ld3.f19_g16_rx1.A",
-                             "ERR_Ld3.f45_g37_rx1.A",
-                             "SMS_D_Ln9_Mmpi-serial.f19_g16_rx1.A")
-                            ),
-
     "acme_developer" : ("acme_land_developer",
                         ("ERS.f19_g16_rx1.A",
                          "ERS.f45_g37_rx1.DTEST",
@@ -194,16 +200,16 @@ def get_full_test_names(testargs, machine, compiler):
 
     Testargs can be categories or test names and support the NOT symbol '^'
 
-    >>> get_full_test_names(["acme_tiny"], "melvin", "gnu")
+    >>> get_full_test_names(["cime_tiny"], "melvin", "gnu")
     ['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu']
 
-    >>> get_full_test_names(["acme_tiny", "PEA_P1_M.f45_g37_rx1.A"], "melvin", "gnu")
+    >>> get_full_test_names(["cime_tiny", "PEA_P1_M.f45_g37_rx1.A"], "melvin", "gnu")
     ['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'PEA_P1_M.f45_g37_rx1.A.melvin_gnu']
 
     >>> get_full_test_names(['ERS.f19_g16_rx1.A', 'NCK.f19_g16_rx1.A', 'PEA_P1_M.f45_g37_rx1.A'], "melvin", "gnu")
     ['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'PEA_P1_M.f45_g37_rx1.A.melvin_gnu']
 
-    >>> get_full_test_names(["acme_tiny", "^NCK.f19_g16_rx1.A"], "melvin", "gnu")
+    >>> get_full_test_names(["cime_tiny", "^NCK.f19_g16_rx1.A"], "melvin", "gnu")
     ['ERS.f19_g16_rx1.A.melvin_gnu']
     """
     expect(machine is not None, "Must define a machine")