per #1657, use met_util.mkdir_p instead of os.makedirs to create dire…

…ctories to prevent issues with parallel runs of METplus creating directories

metplus/util/config_metplus.py

@@ -251,8 +251,7 @@ def launch(config_list):
 
     # create final conf directory if it doesn't already exist
     final_conf_dir = os.path.dirname(final_conf)
-    if not os.path.exists(final_conf_dir):
-        os.makedirs(final_conf_dir)
+    util.mkdir_p(final_conf_dir)
 
     # set METPLUS_BASE/PARM_BASE conf so they can be referenced in other confs
     config.set('config', 'METPLUS_BASE', METPLUS_BASE)
@@ -372,10 +371,8 @@ def get_logger(config, sublog=None):
     log_dir = config.getdir('LOG_DIR')
     log_level = config.getstr('config', 'LOG_LEVEL')
 
-    # Check if the directory path for the log file exists, if
-    # not create it.
-    if not os.path.exists(log_dir):
-        util.mkdir_p(log_dir)
+    # Create the log directory if it does not exist
+    util.mkdir_p(log_dir)
 
     if sublog is not None:
         logger = config.log(sublog)

metplus/util/met_util.py

@@ -251,9 +251,7 @@ def handle_tmp_dir(config):
     # create temp dir if it doesn't exist already
     # this will fail if TMP_DIR is not set correctly and
     # env MET_TMP_DIR was not set
-    tmp_dir = config.getdir('TMP_DIR')
-    if not os.path.exists(tmp_dir):
-        os.makedirs(tmp_dir)
+    mkdir_p(config.getdir('TMP_DIR'))
 
 def handle_env_var_config(config, env_var_name, config_name):
     """! If environment variable is set, use that value
@@ -1229,9 +1227,7 @@ def preprocess_file(filename, data_type, config, allow_dir=False):
                 return stagefile
             # if it does not exist, run GempakToCF and return staged nc file
             # Create staging area if it does not exist
-            outdir = os.path.dirname(stagefile)
-            if not os.path.exists(outdir):
-                os.makedirs(outdir, mode=0o0775)
+            mkdir_p(os.path.dirname(stagefile))
 
             # only import GempakToCF if needed
             from ..wrappers import GempakToCFWrapper
@@ -1262,9 +1258,7 @@ def preprocess_file(filename, data_type, config, allow_dir=False):
     # Create staging area directory only if file has compression extension
     if any([os.path.isfile(f'{filename}{ext}')
             for ext in COMPRESSION_EXTENSIONS]):
-        outdir = os.path.dirname(outpath)
-        if not os.path.exists(outdir):
-            os.makedirs(outdir, mode=0o0775)
+        mkdir_p(os.path.dirname(outpath))
 
     # uncompress gz, bz2, or zip file
     if os.path.isfile(filename+".gz"):

metplus/wrappers/command_builder.py

@@ -912,8 +912,7 @@ def write_list_file(self, filename, file_list, output_dir=None):
 
         list_path = os.path.join(list_dir, filename)
 
-        if not os.path.exists(list_dir):
-            os.makedirs(list_dir, mode=0o0775)
+        util.mkdir_p(list_dir)
 
         self.logger.debug("Writing list of filenames...")
         with open(list_path, 'w') as file_handle:
@@ -1008,7 +1007,7 @@ def find_and_check_output_file(self, time_info=None,
         if (not os.path.exists(parent_dir) and
                 not self.c_dict.get('DO_NOT_RUN_EXE', False)):
             self.logger.debug(f"Creating output directory: {parent_dir}")
-            os.makedirs(parent_dir)
+            util.mkdir_p(parent_dir)
 
         if not output_exists or not skip_if_output_exists:
             return True
@@ -1222,8 +1221,7 @@ def get_command(self):
             self.log_error('Must specify path to output file')
             return None
 
-        if not os.path.exists(parent_dir):
-            os.makedirs(parent_dir)
+        util.mkdir_p(parent_dir)
 
         cmd += " " + out_path
 

metplus/wrappers/series_analysis_wrapper.py

@@ -22,7 +22,7 @@
     WRAPPER_CANNOT_RUN = True
     EXCEPTION_ERR = err_msg
 
-from ..util import getlist, get_storms
+from ..util import getlist, get_storms, mkdir_p
 from ..util import do_string_sub, parse_template, get_tags
 from ..util import get_lead_sequence, get_lead_sequence_groups
 from ..util import ti_get_hours_from_lead, ti_get_seconds_from_lead
@@ -1044,8 +1044,7 @@ def generate_animations(self):
 
         animate_dir = os.path.join(self.c_dict['OUTPUT_DIR'],
                                    'series_animate')
-        if not os.path.exists(animate_dir):
-            os.makedirs(animate_dir)
+        mkdir_p(animate_dir)
 
         for group, files in self.c_dict['PNG_FILES'].items():
             # write list of files to a text file

metplus/wrappers/tc_pairs_wrapper.py

@@ -863,8 +863,7 @@ def read_modify_write_file(self, in_csvfile, storm_month, missing_values,
                 @param logger the log where logging is directed
         """
         # create output directory if it does not exist
-        if not os.path.exists(os.path.dirname(out_csvfile)):
-            os.makedirs(os.path.dirname(out_csvfile))
+        mkdir_p(os.path.dirname(out_csvfile))
 
         # Open the output csv file
         out_file = open(out_csvfile, "w", newline='')

metplus/wrappers/tcmpr_plotter_wrapper.py

@@ -226,7 +226,7 @@ def run_all_times(self):
         if not os.path.exists(self.c_dict['OUTPUT_DIR']):
             self.logger.debug("Creating directory: "
                               f"{self.c_dict['OUTPUT_DIR']}")
-            os.makedirs(self.c_dict['OUTPUT_DIR'])
+            mkdir_p(self.c_dict['OUTPUT_DIR'])
 
         self.set_environment_variables()