check whether paths exist

glamod_marine_processing/obs_suite/scripts/_utilities.py

@@ -103,18 +103,9 @@ def __init__(self, process_options, inargs, level, level_prev):
             self.prev_level_path,
             self.level_path,
             self.level_ql_path,
-            self.level_log_path,
-            self.level_invalid_path,
-            self.level_excluded_path,
         ]
 
-        if any([not os.path.isdir(x) for x in data_paths]):
-            logging.error(
-                "Could not find data paths: {}".format(
-                    ",".join([x for x in data_paths if not os.path.isdir(x)])
-                )
-            )
-            sys.exit(1)
+        paths_exist(data_paths)
 
         if len(glob.glob(self.filename)) == 0:
             logging.error(f"L1b header files not found: {self.filename}")
@@ -150,3 +141,16 @@ def clean_level(filenames):
         except Exception:
             logging.warning(f"Could not remove previous file: {filename}")
             pass
+
+def paths_exist(data_paths):
+    """Check whether path(s) exist(s)."""
+    if not isinstance(data_paths, list):
+        data_paths = [data_paths]
+    exit = False
+    for data_path in data_paths:
+        if not os.path.isdir(data_path):
+            exit = True
+            logging.error(f"Could not find data paths: {data_path}")
+    if exit is True:
+        sys.exit(1)
+

glamod_marine_processing/obs_suite/scripts/level1a.py

@@ -67,7 +67,7 @@
 import numpy as np
 import pandas as pd
 import simplejson
-from _utilities import FFS, clean_level, date_handler, script_setup
+from _utilities import FFS, clean_level, date_handler, paths_exist, script_setup
 from cdm_reader_mapper import cdm_mapper as cdm
 from cdm_reader_mapper import mdf_reader, metmetpy
 from cdm_reader_mapper.common import pandas_TextParser_hdlr
@@ -120,9 +120,11 @@ def write_out_junk(dataObj, filename):
 
 L0_filename = os.path.join(params.prev_level_path, params.filename)
 if not os.path.isfile(L0_filename):
-    logging.error(f"L0 file not found: {L0_filename}")
+    logging.error(f"Could not find data input file: {L0_filename}")
     sys.exit(1)
 
+paths_exist([params.level_invalid_path, params.level_excluded_path])
+
 # CLEAN PREVIOUS L1A PRODUCTS AND SIDE FILES ----------------------------------
 L1a_prods = glob.glob(
     os.path.join(params.level_path, "*" + FFS + params.fileID + ".psv")

glamod_marine_processing/obs_suite/scripts/level1b.py

@@ -65,6 +65,7 @@
     clean_level,
     date_handler,
     delimiter,
+    paths_exist,
     script_setup,
     table_to_csv,
 )
@@ -99,10 +100,7 @@
 )
 
 logging.info(f"Setting corrections path to {L1b_main_corrections}")
-if not os.path.isdir(L1b_main_corrections):
-    logging.error(f"Could not find data paths: {L1b_main_corrections}")
-    sys.exit()
-
+paths_exist(L1b_main_corrections)
 
 # Clean previous L1a products and side files ----------------------------------
 L1b_prods = glob.glob(os.path.join(params.level_path, "*-" + params.fileID + ".psv"))

glamod_marine_processing/obs_suite/scripts/level1c.py

@@ -106,7 +106,7 @@
 import numpy as np
 import pandas as pd
 import simplejson
-from _utilities import FFS, clean_level, date_handler, script_setup, table_to_csv
+from _utilities import FFS, clean_level, date_handler, paths_exist, script_setup, table_to_csv
 from cdm_reader_mapper import cdm_mapper as cdm
 
 reload(logging)  # This is to override potential previous config of logging
@@ -235,14 +235,12 @@ def process_table(table_df, table_name):
 
 params = script_setup([], args, "level1c", "level1b")
 
-level_invalid_path = os.path.join(
-    params.release_path, "level1c", "invalid", params.sid_dck
-)
-
 id_validation_path = os.path.join(
     params.data_path, params.release, "NOC_ANC_INFO", "json_files"
 )
 
+paths_exist([id_validation_path, params.level_invalid_path])
+
 # Clean previous L1c products and side files ----------------------------------
 level_prods = glob.glob(
     os.path.join(params.level_path, "*" + FFS + params.fileID + ".psv")

glamod_marine_processing/obs_suite/scripts/level1d.py

@@ -70,6 +70,7 @@
     clean_level,
     date_handler,
     delimiter,
+    paths_exist,
     script_setup,
     table_to_csv,
 )
@@ -192,9 +193,7 @@ def process_table(table_df, table_name):
 scratch_ = os.path.join(params.release_path, level, "scratch")
 scratch_path = os.path.join(scratch_, params.sid_dck)
 os.makedirs(scratch_path, exist_ok=True)
-
-if not os.path.isdir(params.level_log_path):
-    logging.error(f"Could not find data paths: {params.level_log_path}")
+paths_exist(params.level_log_path)
 
 md_avail = True if not params.md_not_avail else False
 

glamod_marine_processing/obs_suite/scripts/level1e.py

@@ -130,7 +130,7 @@
 import numpy as np
 import pandas as pd
 import simplejson
-from _utilities import FFS, clean_level, date_handler, script_setup, table_to_csv
+from _utilities import FFS, clean_level, date_handler, paths_exist, script_setup, table_to_csv
 from cdm_reader_mapper import cdm_mapper as cdm
 
 reload(logging)  # This is to override potential previous config of logging
@@ -422,9 +422,7 @@ def process_table(table_df, table_name):
 qc_path = os.path.join(params.data_path, params.release, "metoffice_qc", "base")
 
 # Check we have all the dirs!
-if not os.path.isdir(qc_path):
-    logging.error(f"Could not find data paths: {qc_path}")
-    sys.exit(1)
+paths_exist(qc_path)
 
 # Check we have QC files!
 logging.info(f"Using qc files in {qc_path}")

glamod_marine_processing/obs_suite/scripts/level2.py

@@ -51,7 +51,7 @@
 from importlib import reload
 from pathlib import Path
 
-from _utilities import clean_level, script_setup
+from _utilities import clean_level, paths_exist, script_setup
 from cdm_reader_mapper import cdm_mapper as cdm
 
 reload(logging)  # This is to override potential previous config of logging
@@ -89,9 +89,7 @@ def copyfiles(pattern, dest, mode="excluded"):
 left_min_period = 1600
 right_max_period = 2100
 
-if not os.path.isdir(params.prev_level_path):
-    logging.error(f"Could not find data paths: {params.prev_level_path}")
-    sys.exit(1)
+paths_exist([params.level_excluded_path, params.level_reports_path])
 
 # Clean previous L2 data and report subdirs -----------------------------------
 L2_prods = glob.glob(os.path.join(params.level_path, "*.psv"))