From 90c3f7eebaf3eedfe0308cbf7c3d54d5bca9237b Mon Sep 17 00:00:00 2001 From: Patrick Worley Date: Thu, 12 Oct 2017 22:44:32 -0400 Subject: [PATCH 1/6] Disable performance archiving if timing dir. can not be written into Currently, if the directory where performance data and associated provenance data are to be saved already exists or can not be created or populated, then the job aborts. This change allows the job to proceed, simply disabling the performance data archiving. This will allow the use of default locations for saving the performance data that are legal for users in the E3SM project but which will not be legal for E3SM users who are not part of the project. These external users can define alternative locations in their job cases. [BFB] --- scripts/lib/CIME/provenance.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py index 8da5db622e1..ba2709e8b71 100644 --- a/scripts/lib/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -96,7 +96,7 @@ def save_build_provenance(case, lid=None): def _save_prerun_timing_acme(case, lid): timing_dir = case.get_value("SAVE_TIMING_DIR") if timing_dir is None or not os.path.isdir(timing_dir): - logger.warning("SAVE_TIMING_DIR '%s' is not valid. ACME requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive timings" % timing_dir) + logger.warning("SAVE_TIMING_DIR {} is not valid. ACME requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive timings".format(timing_dir)) return logger.info("timing dir is {}".format(timing_dir)) @@ -106,10 +106,16 @@ def _save_prerun_timing_acme(case, lid): cimeroot = case.get_value("CIMEROOT") base_case = case.get_value("CASE") full_timing_dir = os.path.join(timing_dir, "performance_archive", getpass.getuser(), base_case, lid) - expect(not os.path.exists(full_timing_dir), "{} already exists".format(full_timing_dir)) + if os.path.exists(full_timing_dir): + logger.warning("{} already exists. Skipping archive of timing data and associated provenance".format(full_timing_dir)) + return + + try: + os.makedirs(full_timing_dir) + except OSError: + logger.warning("{} can not be created. Skipping archive of timing data and associated provenance".format(full_timing_dir)) + return - os.makedirs(full_timing_dir) - expect(os.path.exists(full_timing_dir), "{} does not exists".format(full_timing_dir)) mach = case.get_value("MACH") compiler = case.get_value("COMPILER") From 74dbe10c0e57da650a820ca4cc5f3d23fe69ec83 Mon Sep 17 00:00:00 2001 From: Patrick Worley Date: Fri, 13 Oct 2017 16:22:07 -0400 Subject: [PATCH 2/6] Make similar changes in _save_postrun_timing_acme function The first commit only made the required changes in _save_prerun_acme_timing. These changes to _save_postrun_timing_acme also return rather than abort when the timing directory can not be written into. A message is not output as it should have been output during _save_prerun_acme_timing. [BFB] --- scripts/lib/CIME/provenance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py index ba2709e8b71..dfa5c02d51e 100644 --- a/scripts/lib/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -261,13 +261,15 @@ def _save_postrun_timing_acme(case, lid): touch(os.path.join(caseroot, "timing", timing_saved_file)) if timing_dir is None or not os.path.isdir(timing_dir): - logger.warning("SAVE_TIMING_DIR '%s' is not valid. ACME requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive timings" % timing_dir) return mach = case.get_value("MACH") base_case = case.get_value("CASE") full_timing_dir = os.path.join(timing_dir, "performance_archive", getpass.getuser(), base_case, lid) + if not os.path.isdir(full_timing_dir): + return + # Kill mach_syslog job_id = _get_batch_job_id_for_syslog(case) if job_id is not None: From 1b9d3567f8840dcde6a334081a9cce43fa492b9e Mon Sep 17 00:00:00 2001 From: Patrick Worley Date: Fri, 13 Oct 2017 17:08:20 -0400 Subject: [PATCH 3/6] Modifying warning message when archiving of timing data is disabled Modified warning message when archiving of timing data is disabled. [BFB] --- scripts/lib/CIME/provenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py index dfa5c02d51e..ad759da7802 100644 --- a/scripts/lib/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -96,7 +96,7 @@ def save_build_provenance(case, lid=None): def _save_prerun_timing_acme(case, lid): timing_dir = case.get_value("SAVE_TIMING_DIR") if timing_dir is None or not os.path.isdir(timing_dir): - logger.warning("SAVE_TIMING_DIR {} is not valid. ACME requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive timings".format(timing_dir)) + logger.warning("SAVE_TIMING_DIR {} is not valid. E3SM requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive of timing data".format(timing_dir)) return logger.info("timing dir is {}".format(timing_dir)) From af07b8825edde6953b584d3186275f4ac75054a8 Mon Sep 17 00:00:00 2001 From: Patrick Worley Date: Fri, 13 Oct 2017 17:31:36 -0400 Subject: [PATCH 4/6] Modifying warning message when archiving of timing data is disabled (2) Modified warning message when archiving of timing data is disabled again. [BFB] --- scripts/lib/CIME/provenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py index ad759da7802..f86c000b057 100644 --- a/scripts/lib/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -96,7 +96,7 @@ def save_build_provenance(case, lid=None): def _save_prerun_timing_acme(case, lid): timing_dir = case.get_value("SAVE_TIMING_DIR") if timing_dir is None or not os.path.isdir(timing_dir): - logger.warning("SAVE_TIMING_DIR {} is not valid. E3SM requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive of timing data".format(timing_dir)) + logger.warning("SAVE_TIMING_DIR {} is not valid. E3SM requires a valid SAVE_TIMING_DIR to be set in order to archive timings. Skipping archive of timing data.".format(timing_dir)) return logger.info("timing dir is {}".format(timing_dir)) From 5d8f11b9cbfbd79d7c884b4d0af417be0ee2ecf3 Mon Sep 17 00:00:00 2001 From: Patrick Worley Date: Fri, 13 Oct 2017 17:46:05 -0400 Subject: [PATCH 5/6] Modifying warning message when archiving of timing data is disabled (3) Modified warning message when archiving of timing data is disabled (third edit). [BFB] --- scripts/lib/CIME/provenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py index f86c000b057..819edb4a7ab 100644 --- a/scripts/lib/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -113,7 +113,7 @@ def _save_prerun_timing_acme(case, lid): try: os.makedirs(full_timing_dir) except OSError: - logger.warning("{} can not be created. Skipping archive of timing data and associated provenance".format(full_timing_dir)) + logger.warning("{} cannot be created. Skipping archive of timing data and associated provenance".format(full_timing_dir)) return mach = case.get_value("MACH") From 1313e5cfad4a0f436e9f341f221bf45f421ddb18 Mon Sep 17 00:00:00 2001 From: Patrick Worley Date: Fri, 13 Oct 2017 18:04:15 -0400 Subject: [PATCH 6/6] Modifying warning message when archiving of timing data is disabled (4) Modified warning message when archiving of timing data is disabled (fourth edit). [BFB] --- scripts/lib/CIME/provenance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py index 819edb4a7ab..12248daaf4e 100644 --- a/scripts/lib/CIME/provenance.py +++ b/scripts/lib/CIME/provenance.py @@ -107,13 +107,13 @@ def _save_prerun_timing_acme(case, lid): base_case = case.get_value("CASE") full_timing_dir = os.path.join(timing_dir, "performance_archive", getpass.getuser(), base_case, lid) if os.path.exists(full_timing_dir): - logger.warning("{} already exists. Skipping archive of timing data and associated provenance".format(full_timing_dir)) + logger.warning("{} already exists. Skipping archive of timing data and associated provenance.".format(full_timing_dir)) return try: os.makedirs(full_timing_dir) except OSError: - logger.warning("{} cannot be created. Skipping archive of timing data and associated provenance".format(full_timing_dir)) + logger.warning("{} cannot be created. Skipping archive of timing data and associated provenance.".format(full_timing_dir)) return mach = case.get_value("MACH")