From b8b5d304e1c132d2b237610781d82a07995dd430 Mon Sep 17 00:00:00 2001 From: pmandrik Date: Mon, 16 May 2022 22:53:31 +0200 Subject: [PATCH] change delete files strategy --- applets/fff_deleter.py | 17 +++++++++++------ applets/fff_deleter_c2f11_09_01.py | 1 + applets/fff_simulator.py | 1 + utils/makerpm.sh | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/applets/fff_deleter.py b/applets/fff_deleter.py index 696997a..c0c4ee4 100644 --- a/applets/fff_deleter.py +++ b/applets/fff_deleter.py @@ -86,7 +86,7 @@ def stat(x): return collected, collected_paths class FileDeleter(object): - def __init__(self, top, thresholds, report_directory, log, fake=True, app_tag="fff_deleter"): + def __init__(self, top, thresholds, report_directory, log, fake=True, skip_latest=False, app_tag="fff_deleter"): self.top = top self.fake = fake self.thresholds = thresholds @@ -95,6 +95,7 @@ def __init__(self, top, thresholds, report_directory, log, fake=True, app_tag="f self.app_tag = app_tag self.log = log self.delay_seconds = 30 + self.skip_latest = skip_latest self.hostname = socket.gethostname() @@ -121,7 +122,7 @@ def rename(self, f): return fn - def delete(self, f): + def overwrite(self, f): if not f.endswith(".deleted"): return f @@ -137,7 +138,7 @@ def delete(self, f): return f - def unlink(self, f, json=False): + def delete(self, f, json=False): if not f.endswith(".deleted"): return f @@ -205,6 +206,8 @@ def do_the_cleanup(self): collected, collected_paths = collect(self.top, self.log) self.log.info("Done file collection, took %.03fs.", time.time() - start) + latest_run = collected[-1][0][0] + file_count = len(collected) # stopSizeDelete can still be positive after this @@ -214,6 +217,7 @@ def do_the_cleanup(self): start_cleanup = time.time() for entry in collected: sort_key, fp, fsize, ftime = entry + if self.skip_latest and latest_run == sort_key: continue # do not want to # unlink file and json older than 2 days # this has no effect on thresholds, but affects performance @@ -221,7 +225,7 @@ def do_the_cleanup(self): if fsize == 0 and age >= 2*24*60*60 and fp.endswith(".deleted"): # remove empty and old files # no one uses them anymore... - self.unlink(fp, json=True) + self.delete(fp, json=True) if stopSizeRename <= 0: break @@ -230,11 +234,11 @@ def do_the_cleanup(self): stopSizeRename -= fsize if fp.endswith(".deleted") and stopSizeDelete > 0: - # delete the files which have been previously marked + # overwrite the files which have been previously marked with dummy # and we have disk over-usage stopSizeDelete -= fsize - self.delete(fp) + self.overwrite(fp) elif fp.endswith(".deleted"): # already renamed, do nothing pass @@ -244,6 +248,7 @@ def do_the_cleanup(self): if self.thresholds.has_key("delete_folders") and self.thresholds["delete_folders"]: for entry in collected_paths: + if self.skip_latest and str(latest_run) in entry.path : continue # check if empty - we don't non-empty dirs # empty as in a 'no stream files left to truncate' sense diff --git a/applets/fff_deleter_c2f11_09_01.py b/applets/fff_deleter_c2f11_09_01.py index 44c9ecf..e5de7c7 100644 --- a/applets/fff_deleter_c2f11_09_01.py +++ b/applets/fff_deleter_c2f11_09_01.py @@ -26,3 +26,4 @@ def __run__(opts, **kwargs): service.delay_seconds = 30 service.run_greenlet() + diff --git a/applets/fff_simulator.py b/applets/fff_simulator.py index e07e989..f919b57 100644 --- a/applets/fff_simulator.py +++ b/applets/fff_simulator.py @@ -397,6 +397,7 @@ def output_join(f): else: log.info("Files for this lumi (%06d) will be skipped (to simulate holes in delivery)", play_lumi) + return # files are rather removed gently by FileDeleter greenlet self.manager.register_files_for_cleanup(run, play_lumi, written_files) class RunManager(object): diff --git a/utils/makerpm.sh b/utils/makerpm.sh index 08ce4de..6f87f29 100755 --- a/utils/makerpm.sh +++ b/utils/makerpm.sh @@ -11,7 +11,7 @@ cd $BUILDDIR cat > fff-dqmtools.spec <