Feature 1355 ioda (#1587)

* #1355 Added makefile for ioda2.nc * #1355 Added ioda2nc * #1355 Added unit_ioda2nc.xml * #1355 Added yyyymmddThhmmss_to_unix and is_yyyymmddThhmmss * #1355 Added yyyymmddThhmmss_to_unix and is_yyyymmddThhmmss * #1355 Added parse_conf_metadata_map and parse_conf_obs_name_map * #1355 Added conf_key_obs_name_map, conf_key_metadata_map, and conf_key_missing_thresh * #1355 Exception handlijng at get_att_value_chars * #1355 Initial release * #1355 Initial release * #1355 Cleanup * #1355 Corretced NC_BYTE value * #1355 Initial release * #1355 Added IODA2NCConfig_efault * #1355 Added IODA2NCConfig_default * #1355 Turn off time_summary * #1355 Changed missing_thresh +-1e9 * #1355 Terminate string * #1355 Corrected echo statement * #1355 Removed unused variable * #1355 Make the string null terminated * #1355 Move the IODA2NCConfig_default to above to avoid merge conflict * #1355 To aoide merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * Per #1355, add .gitignore file for ioda2nc. * Per #1355, had to add test stub for the new ioda2nc tool to enable 'make test' to run. * Per #1355, tweak met/scripts/Makefile to NOT ignore the ENABLE_PYTHON configuration option when constructing the list of tests. * Per #1355, ignore the scripts/ioda2nc file. Co-authored-by: John Halley Gotway <[email protected]>
dtcenter · Dec 3, 2020 · 724ec47 · 724ec47
1 parent d7ecff3
commit 724ec47
Show file tree

Hide file tree

Showing 26 changed files with 2,436 additions and 88 deletions.
diff --git a/met/configure.ac b/met/configure.ac
@@ -547,6 +547,26 @@ else
     AC_MSG_NOTICE([grid_stat will not be compiled])
 fi
 
+# ioda2nc
+
+AC_ARG_ENABLE(ioda2nc,
+	  [AS_HELP_STRING([--disable-ioda2nc], [Disable compilation of ioda2nc])],
+  [case "${enableval}" in
+     yes | no ) ENABLE_IODA2NC="${enableval}" ;;
+     *) AC_MSG_ERROR(bad value ${enableval} for --disable-ioda2nc) ;;
+   esac],
+  [ENABLE_IODA2NC="yes"]
+)
+
+AM_CONDITIONAL([ENABLE_IODA2NC], [test "x$ENABLE_IODA2NC" = "xyes"])
+
+if test "x$ENABLE_IODA2NC" = "xyes"; then
+    AC_DEFINE([ENABLE_IODA2NC], [], ["build ioda2nc"])
+    AC_MSG_NOTICE([ioda2nc will be compiled])
+else
+    AC_MSG_NOTICE([ioda2nc will not be compiled])
+fi
+
 # madis2nc
 
 AC_ARG_ENABLE(madis2nc,
@@ -1205,6 +1225,7 @@ AC_CONFIG_FILES([Makefile
                  src/tools/other/lidar2nc/Makefile
                  src/tools/other/gen_vx_mask/Makefile
                  src/tools/other/gis_utils/Makefile
+                 src/tools/other/ioda2nc/Makefile
                  src/tools/other/madis2nc/Makefile
                  src/tools/other/mode_graphics/Makefile
                  src/tools/other/modis_regrid/Makefile

diff --git a/met/data/config/IODA2NCConfig_default b/met/data/config/IODA2NCConfig_default
@@ -0,0 +1,130 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// IODA2NC configuration file.
+//
+// For additional information, see the MET_BASE/config/README file.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// IODA message type
+//
+message_type = [];
+
+//
+// Mapping of message type group name to comma-separated list of values
+// Derive PRMSL only for SURFACE message types
+//
+message_type_group_map = [];
+
+//
+// Mapping of input IODA message types to output message types
+//
+message_type_map = [];
+
+//
+// IODA station ID
+//
+station_id = [];
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Observation time window
+//
+obs_window = {
+   beg = -5400;
+   end =  5400;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Observation retention regions
+//
+mask = {
+   grid = "";
+   poly = "";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Observing location elevation
+//
+elevation_range = {
+   beg =  -1000;
+   end = 100000;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Vertical levels to retain
+//
+level_range = {
+   beg = 1;
+   end = 255;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+//
+// IODA variable names to retain or derive.
+// Use obs_bufr_map to rename variables in the output.
+// If empty or 'all', process all available variables.
+//
+obs_var = [];
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Mapping of input IODA variable names to output variables names.
+// The default IODA map, obs_var_map, is appended to this map.
+//
+obs_name_map = [];
+
+//
+// Default mapping for Metadata.
+//
+metadata_map = [
+   { key = "message_type"; val = "msg_type"; },
+   { key = "station_id";   val = "report_identifier"; },
+   { key = "pressure";     val = "air_pressure,pressure"; },
+   { key = "height";       val = "height,height_above_mean_sea_level"; },
+   { key = "elevation";    val = ""; }
+];
+
+missing_thresh = [ <=-1e9, >=1e9, ==-9999 ];
+
+////////////////////////////////////////////////////////////////////////////////
+
+quality_mark_thresh = 0;
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Time periods for the summarization
+// obs_var (string array) is added and works like grib_code (int array)
+// when use_var_id is enabled and variable names are saved.
+//
+time_summary = {
+  flag = FALSE;
+  raw_data = FALSE;
+  beg = "000000";
+  end = "235959";
+  step = 300;
+  width = 600;
+  grib_code = [];
+  obs_var   = [ "TMP", "WDIR", "RH" ];
+  type = [ "min", "max", "range", "mean", "stdev", "median", "p80" ];
+  vld_freq = 0;
+  vld_thresh = 0.0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+tmp_dir = "/tmp";
+version = "V10.0";
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/met/data/config/Makefile.am b/met/data/config/Makefile.am
@@ -25,6 +25,7 @@ config_DATA = \
 	EnsembleStatConfig_default \
 	GridStatConfig_default \
 	GridDiagConfig_default \
+	IODA2NCConfig_default \
 	Madis2NcConfig_default \
 	MODEAnalysisConfig_default \
 	MODEConfig_default \

diff --git a/met/export.mk b/met/export.mk
@@ -43,6 +43,7 @@ export ENABLE_ASCII2NC
 export ENABLE_ENSEMBLE_STAT
 export ENABLE_GEN_VX_MASK
 export ENABLE_GRID_STAT
+export ENABLE_IODA2NC
 export ENABLE_MADIS2NC
 export ENABLE_MODE
 export ENABLE_MODE_ANALYSIS

diff --git a/met/scripts/.gitignore b/met/scripts/.gitignore
@@ -5,6 +5,7 @@ grib2
 grid_stat
 gis_utils
 gsi_tools
+ioda2nc
 lidar2nc
 madis2nc
 mode_analysis

diff --git a/met/scripts/Makefile b/met/scripts/Makefile
@@ -34,7 +34,7 @@ export TEST_OUT_DIR
    ##
 
 
-TESTS := $(shell grep "define ENABLE_" ../config.h | cut -d" " -f2 | cut -b8- | tr A-Z a-z)
+TESTS := $(shell grep "define ENABLE_" ../config.h | grep -v "ENABLE_PYTHON" | cut -d" " -f2 | cut -b8- | tr A-Z a-z)
 
 
 ########################################################################
@@ -56,6 +56,7 @@ include $(MK_DIR)/grib2.mk
 include $(MK_DIR)/grid_stat.mk
 include $(MK_DIR)/gis_utils.mk
 include $(MK_DIR)/gsi_tools.mk
+include $(MK_DIR)/ioda2nc.mk
 include $(MK_DIR)/lidar2nc.mk
 include $(MK_DIR)/madis2nc.mk
 include $(MK_DIR)/mode.mk

diff --git a/met/scripts/mk/ioda2nc.mk b/met/scripts/mk/ioda2nc.mk
@@ -0,0 +1,26 @@
+
+
+
+########################################################################
+
+
+IODA2NC_EXEC = ${OTHER_DIR}/ioda2nc/ioda2nc
+
+
+########################################################################
+
+
+   ##
+   ##  ioda2nc
+   ##
+   ##     prerequisites:
+   ##
+
+
+ioda2nc: ${IODA2NC_EXEC}
+	@ touch ioda2nc
+
+
+########################################################################
+
+
diff --git a/met/src/basic/vx_cal/time_strings.cc b/met/src/basic/vx_cal/time_strings.cc
@@ -223,6 +223,50 @@ return ( t );
 ////////////////////////////////////////////////////////////////////////
 
 
+unixtime yyyymmddThhmmss_to_unix(const char * text)
+
+{
+
+int month, day, year, hour, minute, second;
+unixtime t;
+char junk[32];
+
+
+substring_vx_cal(text, junk, 0, 3);
+
+year = atoi(junk);
+
+substring_vx_cal(text, junk, 5, 6);
+
+month = atoi(junk);
+
+substring_vx_cal(text, junk, 8, 9);
+
+day = atoi(junk);
+
+substring_vx_cal(text, junk, 11, 12);
+
+hour = atoi(junk);
+
+substring_vx_cal(text, junk, 14, 15);
+
+minute = atoi(junk);
+
+substring_vx_cal(text, junk, 17, 18);
+
+second = atoi(junk);
+
+
+t = mdyhms_to_unix(month, day, year, hour, minute, second);
+
+return ( t );
+
+}
+
+
+////////////////////////////////////////////////////////////////////////
+
+
 unixtime yyyymmdd_hh_to_unix(const char * text)
 
 {
@@ -497,6 +541,7 @@ else if ( strlen(text) == 0 ) {
 }
 else if ( strcmp(text, bad_data_str) == 0 ||
           strcmp(text, na_str      ) == 0 )  t = (unixtime) 0;
+else if ( is_yyyymmddThhmmss (text)       )  t = yyyymmddThhmmss_to_unix (text);
 else if ( is_yyyymmdd_hhmmss (text)       )  t = yyyymmdd_hhmmss_to_unix (text);
 else if ( is_yyyymmdd_hh     (text)       )  t = yyyymmdd_hh_to_unix     (text);
 else if ( is_yyyymmddhhmmss  (text)       )  t = yyyymmddhhmmss_to_unix  (text);
@@ -549,6 +594,7 @@ bool is_datestring(const char * text)
 {
 
 return ( is_yyyymmdd_hhmmss(text) ||
+         is_yyyymmddThhmmss(text) ||
          is_yyyymmdd_hh(text)     ||
          is_yyyymmddhhmmss(text)  ||
          is_yyyymmddhhmm(text)    ||
@@ -633,6 +679,18 @@ return ( check_reg_exp("^[0-9]\\{8\\}_[0-9]\\{6\\}$", text) );
 ////////////////////////////////////////////////////////////////////////
 
 
+bool is_yyyymmddThhmmss(const char * text)
+
+{
+
+return ( check_reg_exp("^[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}T[0-9]\\{2\\}:[0-9]\\{2\\}:[0-9]\\{2\\}Z$", text) );
+
+}
+
+
+////////////////////////////////////////////////////////////////////////
+
+
 int timestring_to_sec(const char * text)
 
 {

diff --git a/met/src/basic/vx_cal/vx_cal.h b/met/src/basic/vx_cal/vx_cal.h
@@ -98,6 +98,7 @@ extern  void         unix_to_yyyymmdd_hhmmss(unixtime, char *);
 extern  ConcatString unix_to_yyyymmdd_hhmmss(unixtime);
 
 extern  unixtime yyyymmdd_hhmmss_to_unix(const char *);
+extern  unixtime yyyymmddThhmmss_to_unix(const char *);
 
 extern  unixtime yyyymmdd_hh_to_unix(const char *);
 
@@ -140,6 +141,8 @@ extern  bool is_yyyymmdd_hh(const char * text);
 
 extern  bool is_yyyymmdd_hhmmss(const char * text);
 
+extern  bool is_yyyymmddThhmmss(const char * text);
+
 extern  bool is_hhmmss(const char * text);
 
 extern  bool is_hh(const char * text);

diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h
@@ -560,6 +560,7 @@ static const char conf_key_message_type_map[]       = "message_type_map";
 static const char conf_key_message_type_group_map[] = "message_type_group_map";
 static const char conf_key_obs_bufr_map[]      = "obs_bufr_map";
 static const char conf_key_obs_bufr_var[]      = "obs_bufr_var";
+static const char conf_key_obs_name_map[]      = "obs_name_map";
 static const char conf_key_obs_prefbufr_map[]  = "obs_prefbufr_map";
 static const char conf_key_key[]               = "key";
 static const char conf_key_val[]               = "val";
@@ -619,6 +620,8 @@ static const char conf_key_gaussian_radius[]   = "gaussian_radius";
 static const char conf_key_trunc_factor[]      = "gaussian_trunc_factor";
 static const char conf_key_eclv_points[]       = "eclv_points";
 static const char conf_key_var_name_map[]      = "var_name_map";
+static const char conf_key_metadata_map[]      = "metadata_map";
+static const char conf_key_missing_thresh[]    = "missing_thresh";
 
 //
 // Entries to override file metadata