diff --git a/glamod_marine_processing/obs_suite/scripts/_utilities.py b/glamod_marine_processing/obs_suite/scripts/_utilities.py index 74d25429..554a2b2c 100755 --- a/glamod_marine_processing/obs_suite/scripts/_utilities.py +++ b/glamod_marine_processing/obs_suite/scripts/_utilities.py @@ -103,18 +103,9 @@ def __init__(self, process_options, inargs, level, level_prev): self.prev_level_path, self.level_path, self.level_ql_path, - self.level_log_path, - self.level_invalid_path, - self.level_excluded_path, ] - if any([not os.path.isdir(x) for x in data_paths]): - logging.error( - "Could not find data paths: {}".format( - ",".join([x for x in data_paths if not os.path.isdir(x)]) - ) - ) - sys.exit(1) + paths_exist(data_paths) if len(glob.glob(self.filename)) == 0: logging.error(f"L1b header files not found: {self.filename}") @@ -150,3 +141,16 @@ def clean_level(filenames): except Exception: logging.warning(f"Could not remove previous file: {filename}") pass + +def paths_exist(data_paths): + """Check whether path(s) exist(s).""" + if not isinstance(data_paths, list): + data_paths = [data_paths] + exit = False + for data_path in data_paths: + if not os.path.isdir(data_path): + exit = True + logging.error(f"Could not find data paths: {data_path}") + if exit is True: + sys.exit(1) + \ No newline at end of file diff --git a/glamod_marine_processing/obs_suite/scripts/level1a.py b/glamod_marine_processing/obs_suite/scripts/level1a.py index 1dbc6b0a..4ce8a70e 100755 --- a/glamod_marine_processing/obs_suite/scripts/level1a.py +++ b/glamod_marine_processing/obs_suite/scripts/level1a.py @@ -67,7 +67,7 @@ import numpy as np import pandas as pd import simplejson -from _utilities import FFS, clean_level, date_handler, script_setup +from _utilities import FFS, clean_level, date_handler, paths_exist, script_setup from cdm_reader_mapper import cdm_mapper as cdm from cdm_reader_mapper import mdf_reader, metmetpy from cdm_reader_mapper.common import pandas_TextParser_hdlr @@ -120,9 +120,11 @@ def write_out_junk(dataObj, filename): L0_filename = os.path.join(params.prev_level_path, params.filename) if not os.path.isfile(L0_filename): - logging.error(f"L0 file not found: {L0_filename}") + logging.error(f"Could not find data input file: {L0_filename}") sys.exit(1) +paths_exist([params.level_invalid_path, params.level_excluded_path]) + # CLEAN PREVIOUS L1A PRODUCTS AND SIDE FILES ---------------------------------- L1a_prods = glob.glob( os.path.join(params.level_path, "*" + FFS + params.fileID + ".psv") diff --git a/glamod_marine_processing/obs_suite/scripts/level1b.py b/glamod_marine_processing/obs_suite/scripts/level1b.py index 691c13fb..85e76bdf 100755 --- a/glamod_marine_processing/obs_suite/scripts/level1b.py +++ b/glamod_marine_processing/obs_suite/scripts/level1b.py @@ -65,6 +65,7 @@ clean_level, date_handler, delimiter, + paths_exist, script_setup, table_to_csv, ) @@ -99,10 +100,7 @@ ) logging.info(f"Setting corrections path to {L1b_main_corrections}") -if not os.path.isdir(L1b_main_corrections): - logging.error(f"Could not find data paths: {L1b_main_corrections}") - sys.exit() - +paths_exist(L1b_main_corrections) # Clean previous L1a products and side files ---------------------------------- L1b_prods = glob.glob(os.path.join(params.level_path, "*-" + params.fileID + ".psv")) diff --git a/glamod_marine_processing/obs_suite/scripts/level1c.py b/glamod_marine_processing/obs_suite/scripts/level1c.py index 2e0d7888..3a5357a4 100755 --- a/glamod_marine_processing/obs_suite/scripts/level1c.py +++ b/glamod_marine_processing/obs_suite/scripts/level1c.py @@ -106,7 +106,7 @@ import numpy as np import pandas as pd import simplejson -from _utilities import FFS, clean_level, date_handler, script_setup, table_to_csv +from _utilities import FFS, clean_level, date_handler, paths_exist, script_setup, table_to_csv from cdm_reader_mapper import cdm_mapper as cdm reload(logging) # This is to override potential previous config of logging @@ -235,14 +235,12 @@ def process_table(table_df, table_name): params = script_setup([], args, "level1c", "level1b") -level_invalid_path = os.path.join( - params.release_path, "level1c", "invalid", params.sid_dck -) - id_validation_path = os.path.join( params.data_path, params.release, "NOC_ANC_INFO", "json_files" ) +paths_exist([id_validation_path, params.level_invalid_path]) + # Clean previous L1c products and side files ---------------------------------- level_prods = glob.glob( os.path.join(params.level_path, "*" + FFS + params.fileID + ".psv") diff --git a/glamod_marine_processing/obs_suite/scripts/level1d.py b/glamod_marine_processing/obs_suite/scripts/level1d.py index 8fff1b12..afaa5480 100755 --- a/glamod_marine_processing/obs_suite/scripts/level1d.py +++ b/glamod_marine_processing/obs_suite/scripts/level1d.py @@ -70,6 +70,7 @@ clean_level, date_handler, delimiter, + paths_exist, script_setup, table_to_csv, ) @@ -192,9 +193,7 @@ def process_table(table_df, table_name): scratch_ = os.path.join(params.release_path, level, "scratch") scratch_path = os.path.join(scratch_, params.sid_dck) os.makedirs(scratch_path, exist_ok=True) - -if not os.path.isdir(params.level_log_path): - logging.error(f"Could not find data paths: {params.level_log_path}") +paths_exist(params.level_log_path) md_avail = True if not params.md_not_avail else False diff --git a/glamod_marine_processing/obs_suite/scripts/level1e.py b/glamod_marine_processing/obs_suite/scripts/level1e.py index 76a64ea6..5422f256 100755 --- a/glamod_marine_processing/obs_suite/scripts/level1e.py +++ b/glamod_marine_processing/obs_suite/scripts/level1e.py @@ -130,7 +130,7 @@ import numpy as np import pandas as pd import simplejson -from _utilities import FFS, clean_level, date_handler, script_setup, table_to_csv +from _utilities import FFS, clean_level, date_handler, paths_exist, script_setup, table_to_csv from cdm_reader_mapper import cdm_mapper as cdm reload(logging) # This is to override potential previous config of logging @@ -422,9 +422,7 @@ def process_table(table_df, table_name): qc_path = os.path.join(params.data_path, params.release, "metoffice_qc", "base") # Check we have all the dirs! -if not os.path.isdir(qc_path): - logging.error(f"Could not find data paths: {qc_path}") - sys.exit(1) +paths_exist(qc_path) # Check we have QC files! logging.info(f"Using qc files in {qc_path}") diff --git a/glamod_marine_processing/obs_suite/scripts/level2.py b/glamod_marine_processing/obs_suite/scripts/level2.py index 848c85ed..2e7d5337 100755 --- a/glamod_marine_processing/obs_suite/scripts/level2.py +++ b/glamod_marine_processing/obs_suite/scripts/level2.py @@ -51,7 +51,7 @@ from importlib import reload from pathlib import Path -from _utilities import clean_level, script_setup +from _utilities import clean_level, paths_exist, script_setup from cdm_reader_mapper import cdm_mapper as cdm reload(logging) # This is to override potential previous config of logging @@ -89,9 +89,7 @@ def copyfiles(pattern, dest, mode="excluded"): left_min_period = 1600 right_max_period = 2100 -if not os.path.isdir(params.prev_level_path): - logging.error(f"Could not find data paths: {params.prev_level_path}") - sys.exit(1) +paths_exist([params.level_excluded_path, params.level_reports_path]) # Clean previous L2 data and report subdirs ----------------------------------- L2_prods = glob.glob(os.path.join(params.level_path, "*.psv"))