diff --git a/scripts/Testing/Testcases/config_tests.xml b/scripts/Testing/Testcases/config_tests.xml
index 1931be4632b..6de7d3e3161 100644
--- a/scripts/Testing/Testcases/config_tests.xml
+++ b/scripts/Testing/Testcases/config_tests.xml
@@ -217,6 +217,16 @@ LAR long term archive test
FALSE
+
+ For testing infra only. Insta-fail build step by failing to init.
+ 1
+ 0
+ ndays
+ 11
+ FALSE
+ FALSE
+
+
For testing infra only. Insta-fail run step.
1
@@ -227,6 +237,16 @@ LAR long term archive test
FALSE
+
+ For testing infra only. Insta-fail run step via exception.
+ 1
+ 0
+ ndays
+ 11
+ FALSE
+ FALSE
+
+
For testing infra only. Insta-pass run step.
1
diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build
index 976b28dd98d..6c99aa7b4f3 100755
--- a/scripts/Tools/case.build
+++ b/scripts/Tools/case.build
@@ -11,6 +11,7 @@ from CIME.case import Case
from CIME.utils import expect, append_status, find_system_test
from CIME.XML.files import Files
from CIME.XML.component import Component
+from CIME.test_status import *
###############################################################################
def parse_command_line(args, description):
@@ -85,7 +86,17 @@ def _main_func(description):
elif(testname is not None):
logging.warn("Building test for %s in directory %s" %
(testname, caseroot))
- test = find_system_test(testname, case)(case)
+ try:
+ # The following line can throw exceptions if the testname is
+ # not found or the test constructor throws. We need to be
+ # sure to leave TestStatus in the appropriate state if that
+ # happens.
+ test = find_system_test(testname, case)(case)
+ except:
+ phase_to_fail = MODEL_BUILD_PHASE if model_only else SHAREDLIB_BUILD_PHASE
+ with TestStatus(test_dir=caseroot) as ts:
+ ts.set_status(phase_to_fail, TEST_FAIL_STATUS, comments="failed to initialize")
+ raise
append_status("case.testbuild starting ",
caseroot=caseroot,sfile="CaseStatus")
diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py
index 19f5d2439e2..48440a4dc94 100644
--- a/utils/python/CIME/SystemTests/system_tests_common.py
+++ b/utils/python/CIME/SystemTests/system_tests_common.py
@@ -491,11 +491,22 @@ def build_phase(self, sharedlib_only=False, model_only=False):
FakeTest.build_phase(self,
sharedlib_only=sharedlib_only, model_only=model_only)
+class TESTRUNFAILEXC(TESTRUNPASS):
+
+ def run_phase(self):
+ raise RuntimeError("Exception from run_phase")
+
class TESTBUILDFAIL(FakeTest):
def build_phase(self, sharedlib_only=False, model_only=False):
if (not sharedlib_only):
- expect(False, "ERROR: Intentional fail for testing infrastructure")
+ expect(False, "Intentional fail for testing infrastructure")
+
+class TESTBUILDFAILEXC(FakeTest):
+
+ def __init__(self, case):
+ FakeTest.__init__(self, case)
+ raise RuntimeError("Exception from init")
class TESTRUNSLOWPASS(FakeTest):
diff --git a/utils/python/CIME/case_setup.py b/utils/python/CIME/case_setup.py
index 4e781618623..4713e70c606 100644
--- a/utils/python/CIME/case_setup.py
+++ b/utils/python/CIME/case_setup.py
@@ -295,12 +295,15 @@ def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, r
def case_setup(case, clean=False, test_mode=False, reset=False):
###############################################################################
caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID")
- test_name = casebaseid if casebaseid is not None else case.get_value("CASE")
- with TestStatus(test_dir=caseroot, test_name=test_name) as ts:
- try:
- _case_setup_impl(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset)
- except:
- ts.set_status(SETUP_PHASE, TEST_FAIL_STATUS)
- raise
- else:
- ts.set_status(SETUP_PHASE, TEST_PASS_STATUS)
+ if case.get_value("TEST"):
+ test_name = casebaseid if casebaseid is not None else case.get_value("CASE")
+ with TestStatus(test_dir=caseroot, test_name=test_name) as ts:
+ try:
+ _case_setup_impl(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset)
+ except:
+ ts.set_status(SETUP_PHASE, TEST_FAIL_STATUS)
+ raise
+ else:
+ ts.set_status(SETUP_PHASE, TEST_PASS_STATUS)
+ else:
+ _case_setup_impl(case, caseroot, casebaseid, clean=clean, test_mode=test_mode, reset=reset)
diff --git a/utils/python/CIME/case_test.py b/utils/python/CIME/case_test.py
index 0175202f6cd..5945b4cc2e8 100644
--- a/utils/python/CIME/case_test.py
+++ b/utils/python/CIME/case_test.py
@@ -13,7 +13,18 @@ def case_test(case, testname=None):
expect(testname is not None, "testname argument not resolved")
logging.warn("Running test for %s" % testname)
- test = find_system_test(testname, case)(case)
+ try:
+ # The following line can throw exceptions if the testname is
+ # not found or the test constructor throws. We need to be
+ # sure to leave TestStatus in the appropriate state if that
+ # happens.
+ test = find_system_test(testname, case)(case)
+ except:
+ caseroot = case.get_value("CASEROOT")
+ with TestStatus(test_dir=caseroot) as ts:
+ ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize")
+ raise
+
success = test.run()
return success
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
index 842e739ddee..606d7be63a4 100755
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -425,7 +425,7 @@ def simple_test(self, expect_works, extra_args):
###########################################################################
if NO_BATCH:
extra_args += " --no-batch"
- cmd = "%s/create_test acme_test_only_pass %s" % (SCRIPT_DIR, extra_args)
+ cmd = "%s/create_test cime_test_only_pass %s" % (SCRIPT_DIR, extra_args)
stat, output, errput = run_cmd(cmd)
if (expect_works):
self.assertEqual(stat, 0, msg="COMMAND '%s' SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (cmd, output, errput, stat))
@@ -479,8 +479,11 @@ class E_TestTestScheduler(TestCreateTestCommon):
def test_a_phases(self):
###########################################################################
# exclude the MEMLEAK tests here.
- tests = update_acme_tests.get_full_test_names(["acme_test_only",
- "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X", "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X"],
+ tests = update_acme_tests.get_full_test_names(["cime_test_only",
+ "^TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X",
+ "^TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X",
+ "^TESTBUILDFAILEXC.f19_g16_rx1.A",
+ "^TESTRUNFAILEXC_Mmpi-serial.f19_g16_rx1.A"],
self._machine, self._compiler)
self.assertEqual(len(tests), 3)
ct = TestScheduler(tests)
@@ -551,15 +554,17 @@ def test_a_phases(self):
###########################################################################
def test_b_full(self):
###########################################################################
- tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler)
+ tests = update_acme_tests.get_full_test_names(["cime_test_only"], self._machine, self._compiler)
test_id="%s-%s" % (self._baseline_name, CIME.utils.get_utc_timestamp())
ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH)
- build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0]
- run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0]
- pass_test = [item for item in tests if "TESTRUNPASS" in item][0]
- mem_fail_test = [item for item in tests if "TESTMEMLEAKFAIL" in item][0]
- mem_pass_test = [item for item in tests if "TESTMEMLEAKPASS" in item][0]
+ build_fail_test = [item for item in tests if "TESTBUILDFAIL." in item][0]
+ build_fail_exc_test = [item for item in tests if "TESTBUILDFAILEXC" in item][0]
+ run_fail_test = [item for item in tests if "TESTRUNFAIL_" in item][0]
+ run_fail_exc_test = [item for item in tests if "TESTRUNFAILEXC" in item][0]
+ pass_test = [item for item in tests if "TESTRUNPASS" in item][0]
+ mem_fail_test = [item for item in tests if "TESTMEMLEAKFAIL" in item][0]
+ mem_pass_test = [item for item in tests if "TESTMEMLEAKPASS" in item][0]
log_lvl = logging.getLogger().getEffectiveLevel()
logging.disable(logging.CRITICAL)
@@ -584,8 +589,16 @@ def test_b_full(self):
self.assertEqual(ts.get_status(CIME.test_scheduler.MODEL_BUILD_PHASE), TEST_FAIL_STATUS)
self.assertTrue("Intentional fail for testing infrastructure" in open(log_file, "r").read(),
"Broken test did not report build error")
+ elif (test_name == build_fail_exc_test):
+ self.assertEqual(ts.get_status(CIME.test_scheduler.SHAREDLIB_BUILD_PHASE), TEST_FAIL_STATUS)
+ self.assertTrue("Exception from init" in open(log_file, "r").read(),
+ "Broken test did not report build error")
elif (test_name == run_fail_test):
self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_FAIL_STATUS)
+ elif (test_name == run_fail_exc_test):
+ self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_FAIL_STATUS)
+ self.assertTrue("Exception from run_phase" in open(log_file, "r").read(),
+ "Broken test did not report build error")
elif (test_name == mem_fail_test):
self.assertEqual(ts.get_status(MEMLEAK_PHASE), TEST_FAIL_STATUS)
self.assertEqual(ts.get_status(CIME.test_scheduler.RUN_PHASE), TEST_PASS_STATUS)
@@ -644,7 +657,7 @@ def assert_num_leftovers(self, test_id=None):
# the testroot (bld/run dump area) and jenkins root
if (test_id is None):
test_id = self._baseline_name
- num_tests_in_tiny = len(update_acme_tests.get_test_suite("acme_test_only_pass"))
+ num_tests_in_tiny = len(update_acme_tests.get_test_suite("cime_test_only_pass"))
jenkins_dirs = glob.glob("%s/*%s*/" % (self._jenkins_root, test_id)) # case dirs
# scratch_dirs = glob.glob("%s/*%s*/" % (self._testroot, test_id)) # blr/run dirs
@@ -665,11 +678,11 @@ def test_jenkins_generic_job(self):
# Generate fresh baselines so that this test is not impacted by
# unresolved diffs
- self.simple_test(True, "-t acme_test_only_pass -g -b %s" % self._baseline_name)
+ self.simple_test(True, "-t cime_test_only_pass -g -b %s" % self._baseline_name)
self.assert_num_leftovers()
build_name = "jenkins_generic_job_pass_%s" % CIME.utils.get_utc_timestamp()
- self.simple_test(True, "-t acme_test_only_pass -b %s" % self._baseline_name, build_name=build_name)
+ self.simple_test(True, "-t cime_test_only_pass -b %s" % self._baseline_name, build_name=build_name)
self.assert_num_leftovers() # jenkins_generic_job should have automatically cleaned up leftovers from prior run
assert_dashboard_has_build(self, build_name)
@@ -677,7 +690,7 @@ def test_jenkins_generic_job(self):
def test_jenkins_generic_job_kill(self):
###########################################################################
build_name = "jenkins_generic_job_kill_%s" % CIME.utils.get_utc_timestamp()
- run_thread = threading.Thread(target=self.threaded_test, args=(False, " -t acme_test_only_slow_pass -b master --baseline-compare=no", build_name))
+ run_thread = threading.Thread(target=self.threaded_test, args=(False, " -t cime_test_only_slow_pass -b master --baseline-compare=no", build_name))
run_thread.daemon = True
run_thread.start()
@@ -768,7 +781,7 @@ def test_update_acme_tests(self):
###########################################################################
# Add some testable stuff to acme tests
pass
- # update_acme_tests._TEST_SUITES["acme_tiny"] = \
+ # update_acme_tests._TEST_SUITES["cime_tiny"] = \
# (None, (("ERS.f19_g16_rx1.A", "jgftestmodtest/test_mod"),
# ("NCK.f19_g16_rx1.A", "jgftestmodtest/test_mod"))
# )
@@ -790,7 +803,7 @@ def test_update_acme_tests_test_mods(self):
# not_my_machine = "%s_jgftest" % machine
# # Add some testable stuff to acme tests
- # update_acme_tests._TEST_SUITES["acme_tiny"] = \
+ # update_acme_tests._TEST_SUITES["cime_tiny"] = \
# (None, (("ERS.f19_g16_rx1.A", "test_mod"),
# ("ERS.f19_g16_rx1.B", "test_mod", machine),
# ("ERS.f19_g16_rx1.C", "test_mod", (machine, not_my_machine)),
@@ -798,7 +811,7 @@ def test_update_acme_tests_test_mods(self):
# "ERS.f19_g16_rx1.E")
# )
- # tests = update_acme_tests.get_test_suite("acme_tiny", compiler="gnu")
+ # tests = update_acme_tests.get_test_suite("cime_tiny", compiler="gnu")
# self.assertEqual(5, len(tests))
# self.assertTrue("ERS.f19_g16_rx1.A.melvin_gnu.test_mod" in tests)
@@ -861,7 +874,7 @@ class TestCimeCase(TestCreateTestCommon):
###########################################################################
def test_cime_case(self):
###########################################################################
- stat, output, errput = run_cmd("%s/create_test acme_test_only -t %s --no-build" % (SCRIPT_DIR, self._baseline_name))
+ stat, output, errput = run_cmd("%s/create_test cime_test_only -t %s --no-build" % (SCRIPT_DIR, self._baseline_name))
self.assertEqual(stat, 0,
msg="COMMAND SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (output, errput, stat))
diff --git a/utils/python/update_acme_tests.py b/utils/python/update_acme_tests.py
index b175eb35aa4..e605f6457c1 100644
--- a/utils/python/update_acme_tests.py
+++ b/utils/python/update_acme_tests.py
@@ -10,31 +10,47 @@
# If testmods are needed, a 2-ple must be provided (test, mods)
# If you want to restrict the test mods to certain machines, than a 3-ple is needed (test, mods, [machines])
_TEST_SUITES = {
- "acme_tiny" : (None,
+ "cime_tiny" : (None,
("ERS.f19_g16_rx1.A",
"NCK.f19_g16_rx1.A")
),
- "acme_test_only_pass" : (None,
+ "cime_test_only_pass" : (None,
("TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A",
"TESTRUNPASS_Mmpi-serial.ne30_g16_rx1.A",
"TESTRUNPASS_Mmpi-serial.f45_g37_rx1.A")
),
- "acme_test_only_slow_pass" : (None,
+ "cime_test_only_slow_pass" : (None,
("TESTRUNSLOWPASS_Mmpi-serial.f19_g16_rx1.A",
"TESTRUNSLOWPASS_Mmpi-serial.ne30_g16_rx1.A",
"TESTRUNSLOWPASS_Mmpi-serial.f45_g37_rx1.A")
),
- "acme_test_only" : (None,
+ "cime_test_only" : (None,
("TESTBUILDFAIL.f19_g16_rx1.A",
+ "TESTBUILDFAILEXC.f19_g16_rx1.A",
"TESTRUNFAIL_Mmpi-serial.f19_g16_rx1.A",
+ "TESTRUNFAILEXC_Mmpi-serial.f19_g16_rx1.A",
"TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A",
"TESTMEMLEAKFAIL_Mmpi-serial.f19_g16.X",
"TESTMEMLEAKPASS_Mmpi-serial.f19_g16.X")
),
+ "cime_developer" : (None,
+ ("NCK_Ld3.f45_g37_rx1.A",
+ "ERI.f45_g37.X",
+ "SEQ_Ln9.f19_g16_rx1.A",
+ "ERS_Ld3.ne30_g16_rx1.A",
+ "ERS_N2_Ld3.f19_g16_rx1.A",
+ "ERR_Ld3.f45_g37_rx1.A",
+ "SMS_D_Ln9_Mmpi-serial.f19_g16_rx1.A")
+ ),
+
+ #
+ # ACME tests below
+ #
+
"acme_runoff_developer" : (None,
("SMS.f19_f19.IM1850CLM45CN",
"SMS.f19_f19.IMCLM45")
@@ -63,16 +79,6 @@
"SMS_D.f19_g16.FC5ATMMODCOSP")
),
- "cime_developer" : (None,
- ("NCK_Ld3.f45_g37_rx1.A",
- "ERI.f45_g37.X",
- "SEQ_Ln9.f19_g16_rx1.A",
- "ERS_Ld3.ne30_g16_rx1.A",
- "ERS_N2_Ld3.f19_g16_rx1.A",
- "ERR_Ld3.f45_g37_rx1.A",
- "SMS_D_Ln9_Mmpi-serial.f19_g16_rx1.A")
- ),
-
"acme_developer" : ("acme_land_developer",
("ERS.f19_g16_rx1.A",
"ERS.f45_g37_rx1.DTEST",
@@ -194,16 +200,16 @@ def get_full_test_names(testargs, machine, compiler):
Testargs can be categories or test names and support the NOT symbol '^'
- >>> get_full_test_names(["acme_tiny"], "melvin", "gnu")
+ >>> get_full_test_names(["cime_tiny"], "melvin", "gnu")
['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu']
- >>> get_full_test_names(["acme_tiny", "PEA_P1_M.f45_g37_rx1.A"], "melvin", "gnu")
+ >>> get_full_test_names(["cime_tiny", "PEA_P1_M.f45_g37_rx1.A"], "melvin", "gnu")
['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'PEA_P1_M.f45_g37_rx1.A.melvin_gnu']
>>> get_full_test_names(['ERS.f19_g16_rx1.A', 'NCK.f19_g16_rx1.A', 'PEA_P1_M.f45_g37_rx1.A'], "melvin", "gnu")
['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'PEA_P1_M.f45_g37_rx1.A.melvin_gnu']
- >>> get_full_test_names(["acme_tiny", "^NCK.f19_g16_rx1.A"], "melvin", "gnu")
+ >>> get_full_test_names(["cime_tiny", "^NCK.f19_g16_rx1.A"], "melvin", "gnu")
['ERS.f19_g16_rx1.A.melvin_gnu']
"""
expect(machine is not None, "Must define a machine")