diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 391508320a5..1db0cd989a5 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -232,15 +232,18 @@ def run_indv(self, suffix="base", st_archive=False): case_st_archive(self._case) def _coupler_log_indicates_run_complete(self): - newestcpllogfile = self._case.get_latest_cpl_log() - logger.debug("Latest Coupler log file is {}".format(newestcpllogfile)) + newestcpllogfiles = self._get_latest_cpl_logs() + logger.debug("Latest Coupler log file(s) {}" .format(newestcpllogfiles)) # Exception is raised if the file is not compressed - try: - if "SUCCESSFUL TERMINATION" in gzip.open(newestcpllogfile, 'rb').read(): - return True - except: - logger.info("{} is not compressed, assuming run failed".format(newestcpllogfile)) - return False + allgood = len(newestcpllogfiles) + for cpllog in newestcpllogfiles: + try: + if "SUCCESSFUL TERMINATION" in gzip.open(cpllog, 'rb').read(): + allgood = allgood - 1 + except: + logger.info("{} is not compressed, assuming run failed".format(cpllog)) + + return allgood==0 def _component_compare_copy(self, suffix): comments = copy(self._case, suffix) @@ -294,33 +297,33 @@ def _check_for_memleak(self): Examine memory usage as recorded in the cpl log file and look for unexpected increases. """ - cpllog = self._case.get_latest_cpl_log() - - memlist = self._get_mem_usage(cpllog) + latestcpllogs = self._get_latest_cpl_logs() + for cpllog in latestcpllogs: + memlist = self._get_mem_usage(cpllog) - with self._test_status: - if len(memlist)<3: - self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test") - else: - finaldate = int(memlist[-1][0]) - originaldate = int(memlist[0][0]) - finalmem = float(memlist[-1][1]) - originalmem = float(memlist[0][1]) - memdiff = -1 - if originalmem > 0: - memdiff = (finalmem - originalmem)/originalmem - tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") - if tolerance is None: - tolerance = 0.1 - expect(tolerance > 0.0, "Bad value for memleak tolerance in test") - if memdiff < 0: + with self._test_status: + if len(memlist)<3: self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test") - elif memdiff < tolerance: - self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) else: - comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format(originalmem, finalmem, finaldate-originaldate) - append_testlog(comment) - self._test_status.set_status(MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment) + finaldate = int(memlist[-1][0]) + originaldate = int(memlist[0][0]) + finalmem = float(memlist[-1][1]) + originalmem = float(memlist[0][1]) + memdiff = -1 + if originalmem > 0: + memdiff = (finalmem - originalmem)/originalmem + tolerance = self._case.get_value("TEST_MEMLEAK_TOLERANCE") + if tolerance is None: + tolerance = 0.1 + expect(tolerance > 0.0, "Bad value for memleak tolerance in test") + if memdiff < 0: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS, comments="insuffiencient data for memleak test") + elif memdiff < tolerance: + self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS) + else: + comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format(originalmem, finalmem, finaldate-originaldate) + append_testlog(comment) + self._test_status.set_status(MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment) def compare_env_run(self, expected=None): """ @@ -339,6 +342,25 @@ def compare_env_run(self, expected=None): return False return True + def _get_latest_cpl_logs(self): + """ + find and return the latest cpl log file in the run directory + """ + coupler_log_path = self._case.get_value("RUNDIR") + cpllogs = glob.glob(os.path.join(coupler_log_path, 'cpl*.log.*')) + lastcpllogs = [] + if cpllogs: + lastcpllogs.append(max(cpllogs, key=os.path.getctime)) + basename = os.path.basename(lastcpllogs[0]) + suffix = basename.split('.',1)[1] + for log in cpllogs: + if log in lastcpllogs: + continue + if log.endswith(suffix): + lastcpllogs.append(log) + + return lastcpllogs + def _compare_baseline(self): """ compare the current test output to a baseline result @@ -354,35 +376,38 @@ def _compare_baseline(self): basecmp_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), baseline_name) # compare memory usage to baseline - newestcpllogfile = self._case.get_latest_cpl_log() - memlist = self._get_mem_usage(newestcpllogfile) - baselog = os.path.join(basecmp_dir, "cpl.log.gz") - if not os.path.isfile(baselog): - # for backward compatibility - baselog = os.path.join(basecmp_dir, "cpl.log") - if os.path.isfile(baselog) and len(memlist) > 3: - blmem = self._get_mem_usage(baselog)[-1][1] - curmem = memlist[-1][1] - diff = (curmem-blmem)/blmem - if(diff < 0.1): - self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) - else: - comment = "Error: Memory usage increase > 10% from baseline" - self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment) - append_testlog(comment) - - # compare throughput to baseline - current = self._get_throughput(newestcpllogfile) - baseline = self._get_throughput(baselog) - #comparing ypd so bigger is better - if baseline is not None and current is not None: - diff = (baseline - current)/baseline - if(diff < 0.25): - self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) - else: - comment = "Error: Computation time increase > 25% from baseline" - self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment) - append_testlog(comment) + newestcpllogfiles = self._get_latest_cpl_logs() + memlist = self._get_mem_usage(newestcpllogfiles[0]) + for cpllog in newestcpllogfiles: + m = re.search(r"(cpl.*.log).*.gz",cpllog) + if m is not None: + baselog = os.path.join(basecmp_dir, m.group(1))+".gz" + if baselog is None or not os.path.isfile(baselog): + # for backward compatibility + baselog = os.path.join(basecmp_dir, "cpl.log") + if os.path.isfile(baselog) and len(memlist) > 3: + blmem = self._get_mem_usage(baselog)[-1][1] + curmem = memlist[-1][1] + diff = (curmem-blmem)/blmem + if(diff < 0.1): + self._test_status.set_status(MEMCOMP_PHASE, TEST_PASS_STATUS) + else: + comment = "Error: Memory usage increase > 10% from baseline" + self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment) + append_testlog(comment) + + # compare throughput to baseline + current = self._get_throughput(cpllog) + baseline = self._get_throughput(baselog) + #comparing ypd so bigger is better + if baseline is not None and current is not None: + diff = (baseline - current)/baseline + if(diff < 0.25): + self._test_status.set_status(THROUGHPUT_PHASE, TEST_PASS_STATUS) + else: + comment = "Error: Computation time increase > 25% from baseline" + self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment) + append_testlog(comment) def _generate_baseline(self): """ @@ -395,6 +420,16 @@ def _generate_baseline(self): status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS baseline_name = self._case.get_value("BASEGEN_CASE") self._test_status.set_status("{}".format(GENERATE_PHASE), status, comments=os.path.dirname(baseline_name)) + basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) + # copy latest cpl log to baseline + # drop the date so that the name is generic + newestcpllogfiles = self._get_latest_cpl_logs() + for cpllog in newestcpllogfiles: + m = re.search(r"(cpl.*.log).*.gz",cpllog) + if m is not None: + baselog = os.path.join(basegen_dir, m.group(1))+".gz" + shutil.copyfile(cpllog, + os.path.join(basegen_dir,baselog)) class FakeTest(SystemTestsCommon): """