diff --git a/python/utils/gcp_utils.py b/python/utils/gcp_utils.py index 0da5e52..cc3e107 100644 --- a/python/utils/gcp_utils.py +++ b/python/utils/gcp_utils.py @@ -312,7 +312,7 @@ def delete_multiple_files( verbose (bool, optional): Whether to log each job's success. Defaults to False. job_complete_for_logging (int, optional): The number of jobs to complete before logging. Defaults to 500. """ - list_of_jobs_args_list = [[file_path] for file_path in files_to_delete] + list_of_jobs_args_list = [[file_path] for file_path in set(files_to_delete)] MultiThreadedJobs().run_multi_threaded_job( workers=workers, diff --git a/python/utils/thread_pool_executor_util.py b/python/utils/thread_pool_executor_util.py index d423458..d703a37 100644 --- a/python/utils/thread_pool_executor_util.py +++ b/python/utils/thread_pool_executor_util.py @@ -27,6 +27,7 @@ def execute_with_retries( try: job_output = function(*job_args_list) job_completed = True + return job_completed, job_output except Exception as e: logging.warning(f"Job failed with error: {e}. Retry {retries + 1}/{max_retries}") retries += 1