Feature dtcenter/METplus-Internal#32 add timestamp to final conf and …

…file_lists (#1763)

.github/parm/test_settings.conf

@@ -8,6 +8,8 @@ LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT}
 
 LOG_METPLUS = {LOG_DIR}/metplus.log
 LOG_TIMESTAMP_TEMPLATE =
+METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf
+FILE_LISTS_DIR = {STAGING_DIR}/file_lists
 
 # also set path to GempakToCF.jar for GEMPAK use cases
 GEMPAKTOCF_JAR = /data/input/GempakToCF.jar

internal_tests/pytests/minimum_pytest.conf

@@ -1,10 +1,9 @@
-[dir]
+[config]
 INPUT_BASE = {ENV[METPLUS_TEST_INPUT_BASE]}
 OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}
 MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]}
 TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]}
 
-[config]
 LOG_LEVEL = DEBUG
 LOG_MET_OUTPUT_TO_METPLUS = no
 LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s
@@ -14,3 +13,5 @@ LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT}
 
 LOG_METPLUS = {LOG_DIR}/metplus.log
 LOG_TIMESTAMP_TEMPLATE =
+METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf
+FILE_LISTS_DIR = {STAGING_DIR}/file_lists

metplus/util/config_metplus.py

@@ -293,6 +293,10 @@ def _set_logvars(config, logger=None):
 
     log_filenametimestamp = date_t.strftime(log_timestamp_template)
 
+    # Adding LOG_TIMESTAMP to the final configuration file.
+    logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp))
+    config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp)
+
     log_dir = config.getdir('LOG_DIR')
 
     # NOTE: LOG_METPLUS or metpluslog is meant to include the absolute path
@@ -302,6 +306,7 @@ def _set_logvars(config, logger=None):
     # if LOG_METPLUS =  unset in the conf file, means NO logging.
     # Also, assUmes the user has included the intended path in LOG_METPLUS.
     user_defined_log_file = None
+    metpluslog = ''
     if config.has_option('config', 'LOG_METPLUS'):
         user_defined_log_file = True
         # strinterp will set metpluslog to '' if LOG_METPLUS =  is unset.
@@ -316,34 +321,6 @@ def _set_logvars(config, logger=None):
         if metpluslog:
             if os.path.basename(metpluslog) == metpluslog:
                 metpluslog = os.path.join(log_dir, metpluslog)
-    else:
-        # No LOG_METPLUS in conf file, so let the code try to set it,
-        # if the user defined the variable LOG_FILENAME_TEMPLATE.
-        # LOG_FILENAME_TEMPLATE is an 'unpublished' variable - no one knows
-        # about it unless you are reading this. Why does this exist ?
-        # It was from my first cycle implementation. I did not want to pull
-        # it out, in case the group wanted a stand alone metplus log filename
-        # template variable.
-
-        # If metpluslog_filename includes a path, python joins it intelligently
-        # Set the metplus log filename.
-        # strinterp will set metpluslog_filename to '' if template is empty
-        if config.has_option('config', 'LOG_FILENAME_TEMPLATE'):
-            metpluslog_filename = config.strinterp(
-                'config',
-                '{LOG_FILENAME_TEMPLATE}',
-                LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp
-            )
-        else:
-            metpluslog_filename = ''
-        if metpluslog_filename:
-            metpluslog = os.path.join(log_dir, metpluslog_filename)
-        else:
-            metpluslog = ''
-
-    # Adding LOG_TIMESTAMP to the final configuration file.
-    logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp))
-    config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp)
 
     # Setting LOG_METPLUS in the configuration object
     # At this point LOG_METPLUS will have a value or '' the empty string.

metplus/util/diff_util.py

@@ -78,8 +78,8 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False):
             if not os.path.isfile(filepath_a):
                 continue
 
-            # skip metplus_final.conf
-            if filepath_a.endswith('metplus_final.conf'):
+            # skip final conf file
+            if 'metplus_final.conf' in os.path.basename(filepath_a):
                 continue
 
             filepath_b = filepath_a.replace(dir_a, dir_b)
@@ -109,8 +109,8 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False):
         for filename in files:
             filepath_b = os.path.join(root, filename)
 
-            # skip metplus_final.conf
-            if filepath_b.endswith('metplus_final.conf'):
+            # skip final conf file
+            if 'metplus_final.conf' in os.path.basename(filepath_b):
                 continue
 
             filepath_a = filepath_b.replace(dir_b, dir_a)

metplus/wrappers/command_builder.py

@@ -901,12 +901,11 @@ def write_list_file(self, filename, file_list, output_dir=None):
             @param filename name of ascii file to write
             @param file_list list of files to write to ascii file
             @param output_dir (Optional) directory to write files. If None,
-             ascii files are written to {STAGING_DIR}/file_lists
+            ascii files are written to {FILE_LIST_DIR}
             @returns path to output file
         """
         if output_dir is None:
-            list_dir = os.path.join(self.config.getdir('STAGING_DIR'),
-                                    'file_lists')
+            list_dir = self.config.getdir('FILE_LISTS_DIR')
         else:
             list_dir = output_dir
 

parm/metplus_config/defaults.conf

@@ -32,10 +32,11 @@ INPUT_BASE = /path/to
 
 OUTPUT_BASE = /path/to
 
-METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf
+METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf.{LOG_TIMESTAMP}
 
 TMP_DIR = {OUTPUT_BASE}/tmp
 STAGING_DIR = {OUTPUT_BASE}/stage
+FILE_LISTS_DIR = {STAGING_DIR}/file_lists.{LOG_TIMESTAMP}
 
 
 ###############################################################################
@@ -75,7 +76,7 @@ OMP_NUM_THREADS = 1
 ###############################################################################
 
 
-LOG_METPLUS = {LOG_DIR}/metplus.log.{LOG_TIMESTAMP_TEMPLATE}
+LOG_METPLUS = {LOG_DIR}/metplus.log.{LOG_TIMESTAMP}
 
 LOG_DIR = {OUTPUT_BASE}/logs