From 724ec478148d0bfa4cf2c19f3adc9b38f2d1acd1 Mon Sep 17 00:00:00 2001 From: hsoh-u Date: Thu, 3 Dec 2020 13:49:27 -0700 Subject: [PATCH] Feature 1355 ioda (#1587) * #1355 Added makefile for ioda2.nc * #1355 Added ioda2nc * #1355 Added unit_ioda2nc.xml * #1355 Added yyyymmddThhmmss_to_unix and is_yyyymmddThhmmss * #1355 Added yyyymmddThhmmss_to_unix and is_yyyymmddThhmmss * #1355 Added parse_conf_metadata_map and parse_conf_obs_name_map * #1355 Added conf_key_obs_name_map, conf_key_metadata_map, and conf_key_missing_thresh * #1355 Exception handlijng at get_att_value_chars * #1355 Initial release * #1355 Initial release * #1355 Cleanup * #1355 Corretced NC_BYTE value * #1355 Initial release * #1355 Added IODA2NCConfig_efault * #1355 Added IODA2NCConfig_default * #1355 Turn off time_summary * #1355 Changed missing_thresh +-1e9 * #1355 Terminate string * #1355 Corrected echo statement * #1355 Removed unused variable * #1355 Make the string null terminated * #1355 Move the IODA2NCConfig_default to above to avoid merge conflict * #1355 To aoide merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * #1355 To avoid a merge conflict * Per #1355, add .gitignore file for ioda2nc. * Per #1355, had to add test stub for the new ioda2nc tool to enable 'make test' to run. * Per #1355, tweak met/scripts/Makefile to NOT ignore the ENABLE_PYTHON configuration option when constructing the list of tests. * Per #1355, ignore the scripts/ioda2nc file. Co-authored-by: John Halley Gotway --- met/configure.ac | 21 + met/data/config/IODA2NCConfig_default | 130 ++ met/data/config/Makefile.am | 1 + met/export.mk | 1 + met/scripts/.gitignore | 1 + met/scripts/Makefile | 3 +- met/scripts/mk/ioda2nc.mk | 26 + met/src/basic/vx_cal/time_strings.cc | 58 + met/src/basic/vx_cal/vx_cal.h | 3 + met/src/basic/vx_config/config_constants.h | 3 + met/src/basic/vx_config/config_util.cc | 114 +- met/src/basic/vx_config/config_util.h | 5 +- met/src/libcode/vx_nc_obs/nc_obs_util.cc | 5 +- met/src/libcode/vx_nc_util/nc_utils.cc | 30 +- met/src/tools/other/Makefile.am | 4 + met/src/tools/other/ioda2nc/.gitignore | 7 + met/src/tools/other/ioda2nc/Makefile.am | 46 + met/src/tools/other/ioda2nc/ioda2nc.cc | 1411 +++++++++++++++++ .../tools/other/ioda2nc/ioda2nc_conf_info.cc | 179 +++ .../tools/other/ioda2nc/ioda2nc_conf_info.h | 80 + met/src/tools/other/pb2nc/pb2nc.cc | 46 +- scripts/fortify/run_fortify_sca.sh | 2 +- test/bin/unit_test.sh | 1 + test/config/IODA2NCConfig_mask | 122 ++ test/config/IODA2NCConfig_summary | 122 ++ test/xml/unit_ioda2nc.xml | 103 ++ 26 files changed, 2436 insertions(+), 88 deletions(-) create mode 100644 met/data/config/IODA2NCConfig_default create mode 100644 met/scripts/mk/ioda2nc.mk create mode 100644 met/src/tools/other/ioda2nc/.gitignore create mode 100644 met/src/tools/other/ioda2nc/Makefile.am create mode 100644 met/src/tools/other/ioda2nc/ioda2nc.cc create mode 100644 met/src/tools/other/ioda2nc/ioda2nc_conf_info.cc create mode 100644 met/src/tools/other/ioda2nc/ioda2nc_conf_info.h create mode 100644 test/config/IODA2NCConfig_mask create mode 100644 test/config/IODA2NCConfig_summary create mode 100644 test/xml/unit_ioda2nc.xml diff --git a/met/configure.ac b/met/configure.ac index 527d0ddd61..104612c287 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -547,6 +547,26 @@ else AC_MSG_NOTICE([grid_stat will not be compiled]) fi +# ioda2nc + +AC_ARG_ENABLE(ioda2nc, + [AS_HELP_STRING([--disable-ioda2nc], [Disable compilation of ioda2nc])], + [case "${enableval}" in + yes | no ) ENABLE_IODA2NC="${enableval}" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --disable-ioda2nc) ;; + esac], + [ENABLE_IODA2NC="yes"] +) + +AM_CONDITIONAL([ENABLE_IODA2NC], [test "x$ENABLE_IODA2NC" = "xyes"]) + +if test "x$ENABLE_IODA2NC" = "xyes"; then + AC_DEFINE([ENABLE_IODA2NC], [], ["build ioda2nc"]) + AC_MSG_NOTICE([ioda2nc will be compiled]) +else + AC_MSG_NOTICE([ioda2nc will not be compiled]) +fi + # madis2nc AC_ARG_ENABLE(madis2nc, @@ -1205,6 +1225,7 @@ AC_CONFIG_FILES([Makefile src/tools/other/lidar2nc/Makefile src/tools/other/gen_vx_mask/Makefile src/tools/other/gis_utils/Makefile + src/tools/other/ioda2nc/Makefile src/tools/other/madis2nc/Makefile src/tools/other/mode_graphics/Makefile src/tools/other/modis_regrid/Makefile diff --git a/met/data/config/IODA2NCConfig_default b/met/data/config/IODA2NCConfig_default new file mode 100644 index 0000000000..d65bd3c4c0 --- /dev/null +++ b/met/data/config/IODA2NCConfig_default @@ -0,0 +1,130 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IODA2NC configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// IODA message type +// +message_type = []; + +// +// Mapping of message type group name to comma-separated list of values +// Derive PRMSL only for SURFACE message types +// +message_type_group_map = []; + +// +// Mapping of input IODA message types to output message types +// +message_type_map = []; + +// +// IODA station ID +// +station_id = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation time window +// +obs_window = { + beg = -5400; + end = 5400; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation retention regions +// +mask = { + grid = ""; + poly = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observing location elevation +// +elevation_range = { + beg = -1000; + end = 100000; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Vertical levels to retain +// +level_range = { + beg = 1; + end = 255; +} + +/////////////////////////////////////////////////////////////////////////////// + +// +// IODA variable names to retain or derive. +// Use obs_bufr_map to rename variables in the output. +// If empty or 'all', process all available variables. +// +obs_var = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input IODA variable names to output variables names. +// The default IODA map, obs_var_map, is appended to this map. +// +obs_name_map = []; + +// +// Default mapping for Metadata. +// +metadata_map = [ + { key = "message_type"; val = "msg_type"; }, + { key = "station_id"; val = "report_identifier"; }, + { key = "pressure"; val = "air_pressure,pressure"; }, + { key = "height"; val = "height,height_above_mean_sea_level"; }, + { key = "elevation"; val = ""; } +]; + +missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; + +//////////////////////////////////////////////////////////////////////////////// + +quality_mark_thresh = 0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Time periods for the summarization +// obs_var (string array) is added and works like grib_code (int array) +// when use_var_id is enabled and variable names are saved. +// +time_summary = { + flag = FALSE; + raw_data = FALSE; + beg = "000000"; + end = "235959"; + step = 300; + width = 600; + grib_code = []; + obs_var = [ "TMP", "WDIR", "RH" ]; + type = [ "min", "max", "range", "mean", "stdev", "median", "p80" ]; + vld_freq = 0; + vld_thresh = 0.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +version = "V10.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/met/data/config/Makefile.am b/met/data/config/Makefile.am index 45894e7ae6..fe9b468a48 100644 --- a/met/data/config/Makefile.am +++ b/met/data/config/Makefile.am @@ -25,6 +25,7 @@ config_DATA = \ EnsembleStatConfig_default \ GridStatConfig_default \ GridDiagConfig_default \ + IODA2NCConfig_default \ Madis2NcConfig_default \ MODEAnalysisConfig_default \ MODEConfig_default \ diff --git a/met/export.mk b/met/export.mk index ec950fd16a..93bd734461 100644 --- a/met/export.mk +++ b/met/export.mk @@ -43,6 +43,7 @@ export ENABLE_ASCII2NC export ENABLE_ENSEMBLE_STAT export ENABLE_GEN_VX_MASK export ENABLE_GRID_STAT +export ENABLE_IODA2NC export ENABLE_MADIS2NC export ENABLE_MODE export ENABLE_MODE_ANALYSIS diff --git a/met/scripts/.gitignore b/met/scripts/.gitignore index 6942034da5..2c32bbda55 100644 --- a/met/scripts/.gitignore +++ b/met/scripts/.gitignore @@ -5,6 +5,7 @@ grib2 grid_stat gis_utils gsi_tools +ioda2nc lidar2nc madis2nc mode_analysis diff --git a/met/scripts/Makefile b/met/scripts/Makefile index f6ea85c551..e9bd0c73ff 100755 --- a/met/scripts/Makefile +++ b/met/scripts/Makefile @@ -34,7 +34,7 @@ export TEST_OUT_DIR ## -TESTS := $(shell grep "define ENABLE_" ../config.h | cut -d" " -f2 | cut -b8- | tr A-Z a-z) +TESTS := $(shell grep "define ENABLE_" ../config.h | grep -v "ENABLE_PYTHON" | cut -d" " -f2 | cut -b8- | tr A-Z a-z) ######################################################################## @@ -56,6 +56,7 @@ include $(MK_DIR)/grib2.mk include $(MK_DIR)/grid_stat.mk include $(MK_DIR)/gis_utils.mk include $(MK_DIR)/gsi_tools.mk +include $(MK_DIR)/ioda2nc.mk include $(MK_DIR)/lidar2nc.mk include $(MK_DIR)/madis2nc.mk include $(MK_DIR)/mode.mk diff --git a/met/scripts/mk/ioda2nc.mk b/met/scripts/mk/ioda2nc.mk new file mode 100644 index 0000000000..8708cc8920 --- /dev/null +++ b/met/scripts/mk/ioda2nc.mk @@ -0,0 +1,26 @@ + + + +######################################################################## + + +IODA2NC_EXEC = ${OTHER_DIR}/ioda2nc/ioda2nc + + +######################################################################## + + + ## + ## ioda2nc + ## + ## prerequisites: + ## + + +ioda2nc: ${IODA2NC_EXEC} + @ touch ioda2nc + + +######################################################################## + + diff --git a/met/src/basic/vx_cal/time_strings.cc b/met/src/basic/vx_cal/time_strings.cc index 4bfe14647d..4f3ce06446 100644 --- a/met/src/basic/vx_cal/time_strings.cc +++ b/met/src/basic/vx_cal/time_strings.cc @@ -223,6 +223,50 @@ return ( t ); //////////////////////////////////////////////////////////////////////// +unixtime yyyymmddThhmmss_to_unix(const char * text) + +{ + +int month, day, year, hour, minute, second; +unixtime t; +char junk[32]; + + +substring_vx_cal(text, junk, 0, 3); + +year = atoi(junk); + +substring_vx_cal(text, junk, 5, 6); + +month = atoi(junk); + +substring_vx_cal(text, junk, 8, 9); + +day = atoi(junk); + +substring_vx_cal(text, junk, 11, 12); + +hour = atoi(junk); + +substring_vx_cal(text, junk, 14, 15); + +minute = atoi(junk); + +substring_vx_cal(text, junk, 17, 18); + +second = atoi(junk); + + +t = mdyhms_to_unix(month, day, year, hour, minute, second); + +return ( t ); + +} + + +//////////////////////////////////////////////////////////////////////// + + unixtime yyyymmdd_hh_to_unix(const char * text) { @@ -497,6 +541,7 @@ else if ( strlen(text) == 0 ) { } else if ( strcmp(text, bad_data_str) == 0 || strcmp(text, na_str ) == 0 ) t = (unixtime) 0; +else if ( is_yyyymmddThhmmss (text) ) t = yyyymmddThhmmss_to_unix (text); else if ( is_yyyymmdd_hhmmss (text) ) t = yyyymmdd_hhmmss_to_unix (text); else if ( is_yyyymmdd_hh (text) ) t = yyyymmdd_hh_to_unix (text); else if ( is_yyyymmddhhmmss (text) ) t = yyyymmddhhmmss_to_unix (text); @@ -549,6 +594,7 @@ bool is_datestring(const char * text) { return ( is_yyyymmdd_hhmmss(text) || + is_yyyymmddThhmmss(text) || is_yyyymmdd_hh(text) || is_yyyymmddhhmmss(text) || is_yyyymmddhhmm(text) || @@ -633,6 +679,18 @@ return ( check_reg_exp("^[0-9]\\{8\\}_[0-9]\\{6\\}$", text) ); //////////////////////////////////////////////////////////////////////// +bool is_yyyymmddThhmmss(const char * text) + +{ + +return ( check_reg_exp("^[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}T[0-9]\\{2\\}:[0-9]\\{2\\}:[0-9]\\{2\\}Z$", text) ); + +} + + +//////////////////////////////////////////////////////////////////////// + + int timestring_to_sec(const char * text) { diff --git a/met/src/basic/vx_cal/vx_cal.h b/met/src/basic/vx_cal/vx_cal.h index 4d644fee6c..45d5b71302 100644 --- a/met/src/basic/vx_cal/vx_cal.h +++ b/met/src/basic/vx_cal/vx_cal.h @@ -98,6 +98,7 @@ extern void unix_to_yyyymmdd_hhmmss(unixtime, char *); extern ConcatString unix_to_yyyymmdd_hhmmss(unixtime); extern unixtime yyyymmdd_hhmmss_to_unix(const char *); +extern unixtime yyyymmddThhmmss_to_unix(const char *); extern unixtime yyyymmdd_hh_to_unix(const char *); @@ -140,6 +141,8 @@ extern bool is_yyyymmdd_hh(const char * text); extern bool is_yyyymmdd_hhmmss(const char * text); +extern bool is_yyyymmddThhmmss(const char * text); + extern bool is_hhmmss(const char * text); extern bool is_hh(const char * text); diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index 01f32e518e..374fb9d125 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -560,6 +560,7 @@ static const char conf_key_message_type_map[] = "message_type_map"; static const char conf_key_message_type_group_map[] = "message_type_group_map"; static const char conf_key_obs_bufr_map[] = "obs_bufr_map"; static const char conf_key_obs_bufr_var[] = "obs_bufr_var"; +static const char conf_key_obs_name_map[] = "obs_name_map"; static const char conf_key_obs_prefbufr_map[] = "obs_prefbufr_map"; static const char conf_key_key[] = "key"; static const char conf_key_val[] = "val"; @@ -619,6 +620,8 @@ static const char conf_key_gaussian_radius[] = "gaussian_radius"; static const char conf_key_trunc_factor[] = "gaussian_trunc_factor"; static const char conf_key_eclv_points[] = "eclv_points"; static const char conf_key_var_name_map[] = "var_name_map"; +static const char conf_key_metadata_map[] = "metadata_map"; +static const char conf_key_missing_thresh[] = "missing_thresh"; // // Entries to override file metadata diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc index 069be378c2..f3ee09ccf1 100644 --- a/met/src/basic/vx_config/config_util.cc +++ b/met/src/basic/vx_config/config_util.cc @@ -266,10 +266,10 @@ ConcatString parse_conf_version(Dictionary *dict) { ConcatString parse_conf_string(Dictionary *dict, const char *conf_key, bool check_empty) { ConcatString s; + const char *method_name = "parse_conf_string() -> "; if(!dict) { - mlog << Error << "\nparse_conf_string() -> " - << "empty dictionary!\n\n"; + mlog << Error << "\n" << method_name << "empty dictionary!\n\n"; exit(1); } @@ -278,7 +278,7 @@ ConcatString parse_conf_string(Dictionary *dict, const char *conf_key, // Check for an empty string if(check_empty && s.empty()) { - mlog << Error << "\nparse_conf_string() -> " + mlog << Error << "\n" << method_name << "The \"" << conf_key << "\" entry (\"" << s << "\") cannot be empty.\n\n"; exit(1); @@ -286,7 +286,7 @@ ConcatString parse_conf_string(Dictionary *dict, const char *conf_key, // Check for embedded whitespace in non-empty strings if(!s.empty() && check_reg_exp(ws_reg_exp, s.c_str()) == true) { - mlog << Error << "\nparse_conf_string() -> " + mlog << Error << "\n" << method_name << "The \"" << conf_key << "\" entry (\"" << s << "\") cannot contain embedded whitespace.\n\n"; exit(1); @@ -298,6 +298,18 @@ ConcatString parse_conf_string(Dictionary *dict, const char *conf_key, /////////////////////////////////////////////////////////////////////////////// +StringArray parse_conf_string_array(Dictionary *dict, const char *conf_key, const char *caller) { + StringArray sa, cur, sid_sa; + if(!dict) { + mlog << Error << "\n" << caller << "empty dictionary!\n\n"; + exit(1); + } + + return dict->lookup_string_array(conf_key); +} + +/////////////////////////////////////////////////////////////////////////////// + GrdFileType parse_conf_file_type(Dictionary *dict) { GrdFileType t = FileType_None; int v; @@ -512,20 +524,14 @@ Dictionary parse_conf_i_vx_dict(Dictionary *dict, int index) { /////////////////////////////////////////////////////////////////////////////// StringArray parse_conf_tc_model(Dictionary *dict, bool error_out) { - StringArray sa; - - if(!dict) { - mlog << Error << "\nparse_conf_tc_model() -> " - << "empty dictionary!\n\n"; - exit(1); - } + const char *method_name = "parse_conf_tc_model() -> "; - sa = dict->lookup_string_array(conf_key_model); + StringArray sa = parse_conf_string_array(dict, conf_key_model, method_name); // Print a warning if AVN appears in the model list for(int i=0; i " + mlog << Warning << "\n" << method_name << "Requesting tropical cyclone model name \"" << sa[i] << "\" will yield no results since \"AVN\" is automatically " << "replaced with \"GFS\" when reading ATCF inputs. Please use " @@ -540,19 +546,13 @@ StringArray parse_conf_tc_model(Dictionary *dict, bool error_out) { /////////////////////////////////////////////////////////////////////////////// StringArray parse_conf_message_type(Dictionary *dict, bool error_out) { - StringArray sa; - - if(!dict) { - mlog << Error << "\nparse_conf_message_type() -> " - << "empty dictionary!\n\n"; - exit(1); - } + const char *method_name = "parse_conf_message_type() -> "; - sa = dict->lookup_string_array(conf_key_message_type); + StringArray sa = parse_conf_string_array(dict, conf_key_message_type, method_name); // Check that at least one message type is provided if(error_out && sa.n() == 0) { - mlog << Error << "\nparse_conf_message_type() -> " + mlog << Error << "\n" << method_name << "At least one message type must be provided.\n\n"; exit(1); } @@ -566,14 +566,9 @@ StringArray parse_conf_sid_list(Dictionary *dict, const char *conf_key) { StringArray sa, cur, sid_sa; ConcatString mask_name; int i; + const char *method_name = "parse_conf_sid_list() -> "; - if(!dict) { - mlog << Error << "\nparse_conf_sid_list() -> " - << "empty dictionary!\n\n"; - exit(1); - } - - sa = dict->lookup_string_array(conf_key); + sa = parse_conf_string_array(dict, conf_key, method_name); // Parse station ID's to exclude from each entry for(i=0; i " + mlog << Debug(4) << method_name << "Station ID \"" << conf_key << "\" list contains " << sid_sa.n() << " entries.\n"; @@ -772,15 +767,9 @@ vector parse_conf_llpnt_mask(Dictionary *dict) { /////////////////////////////////////////////////////////////////////////////// StringArray parse_conf_obs_qty(Dictionary *dict) { - StringArray sa; + const char *method_name = "parse_conf_obs_qty() -> "; - if(!dict) { - mlog << Error << "\nparse_conf_obs_qty() -> " - << "empty dictionary!\n\n"; - exit(1); - } - - sa = dict->lookup_string_array(conf_key_obs_qty); + StringArray sa = parse_conf_string_array(dict, conf_key_obs_qty, method_name); return(sa); } @@ -994,15 +983,15 @@ void parse_add_conf_key_value_map( map parse_conf_key_value_map( - Dictionary *dict, const char *conf_key_map_name) { + Dictionary *dict, const char *conf_key_map_name, const char *caller) { Dictionary *msg_typ_dict = (Dictionary *) 0; map m; ConcatString key, val; int i; + const char *method_name = (0 != caller) ? caller : "parse_conf_key_value_map() -> "; if(!dict) { - mlog << Error << "\nparse_conf_key_value_type_map() -> " - << "empty dictionary!\n\n"; + mlog << Error << "\n" << method_name << "empty dictionary!\n\n"; exit(1); } @@ -1017,7 +1006,7 @@ map parse_conf_key_value_map( val = (*msg_typ_dict)[i]->dict_value()->lookup_string(conf_key_val); if(m.count(key) >= 1) { - mlog << Warning << "\nparse_conf_key_value_type_map() -> " + mlog << Warning << "\n" << method_name << "found multiple entries for key \"" << key << "\"!\n\n"; } @@ -1029,21 +1018,18 @@ map parse_conf_key_value_map( return(m); } -/////////////////////////////////////////////////////////////////////////////// - -map parse_conf_message_type_map(Dictionary *dict) { - return parse_conf_key_value_map(dict, conf_key_message_type_map); -} /////////////////////////////////////////////////////////////////////////////// -map parse_conf_message_type_group_map(Dictionary *dict) { +map parse_conf_key_values_map( + Dictionary *dict, const char *conf_key, const char *caller) { + StringArray sa; map cs_map; map::const_iterator it; map sa_map; - StringArray sa; + const char *method_name = (0 != caller) ? caller : "parse_conf_key_values_map() -> "; - cs_map = parse_conf_key_value_map(dict, conf_key_message_type_group_map); + cs_map = parse_conf_key_value_map(dict, conf_key, method_name); // Convert input comma-separated strings to StringArray for(it=cs_map.begin(); it!= cs_map.end(); it++) { @@ -1054,6 +1040,27 @@ map parse_conf_message_type_group_map(Dictionary *dict return sa_map; } + +/////////////////////////////////////////////////////////////////////////////// + +map parse_conf_message_type_map(Dictionary *dict) { + return parse_conf_key_value_map(dict, conf_key_message_type_map); +} + +/////////////////////////////////////////////////////////////////////////////// + +map parse_conf_message_type_group_map(Dictionary *dict) { + const char *method_name = "parse_conf_message_type_group_map() -> "; + return parse_conf_key_values_map(dict, conf_key_message_type_group_map, method_name); +} + +/////////////////////////////////////////////////////////////////////////////// + +map parse_conf_metadata_map(Dictionary *dict) { + const char *method_name = "parse_conf_metadata_map() -> "; + return parse_conf_key_values_map(dict, conf_key_metadata_map, method_name); +} + /////////////////////////////////////////////////////////////////////////////// map parse_conf_obs_bufr_map(Dictionary *dict) { @@ -1064,6 +1071,13 @@ map parse_conf_obs_bufr_map(Dictionary *dict) { /////////////////////////////////////////////////////////////////////////////// +map parse_conf_obs_name_map(Dictionary *dict) { + const char *method_name = "parse_conf_obs_name_map() -> "; + return parse_conf_key_value_map(dict, conf_key_obs_name_map); +} + +/////////////////////////////////////////////////////////////////////////////// + void BootInfo::clear() { interval = BootIntervalType_None; rep_prop = bad_data_double; diff --git a/met/src/basic/vx_config/config_util.h b/met/src/basic/vx_config/config_util.h index 57910638cb..f526165a93 100644 --- a/met/src/basic/vx_config/config_util.h +++ b/met/src/basic/vx_config/config_util.h @@ -46,13 +46,16 @@ extern NumArray parse_conf_eclv_points(Dictionary *dict); extern ClimoCDFInfo parse_conf_climo_cdf(Dictionary *dict); extern TimeSummaryInfo parse_conf_time_summary(Dictionary *dict); extern map parse_conf_key_value_map( - Dictionary *dict, const char *conf_key_map_name); + Dictionary *dict, const char *conf_key_map_name, const char *caller=0); extern map parse_conf_message_type_map(Dictionary *dict); extern map parse_conf_message_type_group_map(Dictionary *dict); +extern map parse_conf_metadata_map(Dictionary *dict); extern map parse_conf_obs_bufr_map(Dictionary *dict); +extern map + parse_conf_obs_name_map(Dictionary *dict); extern BootInfo parse_conf_boot(Dictionary *dict); extern RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out = default_dictionary_error_out); extern InterpInfo parse_conf_interp(Dictionary *dict, const char *); diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.cc b/met/src/libcode/vx_nc_obs/nc_obs_util.cc index de66d7cc6e..2a0b5561e0 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -1282,19 +1282,20 @@ int write_nc_string_array (NcVar *ncVar, StringArray &strArray, const int str_le // Initialize data_buf for (int indexX=0; indexXgetValues(att_value); - value = att_value; + if (attType == NC_CHAR || attType == NC_STRING) { + try { + string att_value; + att->getValues(att_value); + value = att_value; + } + catch (exceptions::NcChar ex) { + value = ""; + // Handle netCDF::exceptions::NcChar: NetCDF: Attempt to convert between text & numbers + mlog << Warning << "\n" << method_name + << "Exception: " << ex.what() << "\n" + << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" + << GET_NC_TYPE_NAME_P(att) << " type).\n" + << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; + } } else { // MET-788: to handle a custom modified NetCDF mlog << Error << "\n" << method_name << "Please convert data type of \"" << GET_NC_NAME_P(att) - << "\" to NC_CHAR type.\n\n"; + << "\" " << GET_NC_TYPE_NAME_P(att) << " to NC_CHAR type.\n\n"; exit(1); } status = true; @@ -474,12 +485,7 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, att = get_nc_att(var, att_name); // Look for a match - if(IS_VALID_NC_P(att)) { - string attr_value; - att->getValues(attr_value); - att_val = attr_value.c_str(); - status = true; - } + status = get_att_value_chars(att, att_val); if (att) delete att; return(status); @@ -1117,7 +1123,7 @@ double get_nc_time(NcVar * var, const int index) { break; case NC_BYTE: var->getVar(start, count, &vb); - k = (double)vs; + k = (double)vb; break; case NC_INT: var->getVar(start, count, &vi); diff --git a/met/src/tools/other/Makefile.am b/met/src/tools/other/Makefile.am index 8239c39f5a..3f56c89a18 100644 --- a/met/src/tools/other/Makefile.am +++ b/met/src/tools/other/Makefile.am @@ -82,6 +82,10 @@ if ENABLE_SHIFT_DATA_PLANE SUBDIRS += shift_data_plane endif +if ENABLE_IODA2NC + SUBDIRS += ioda2nc +endif + SUBDIRS += grid_diag MAINTAINERCLEANFILES = Makefile.in diff --git a/met/src/tools/other/ioda2nc/.gitignore b/met/src/tools/other/ioda2nc/.gitignore new file mode 100644 index 0000000000..0d392220b0 --- /dev/null +++ b/met/src/tools/other/ioda2nc/.gitignore @@ -0,0 +1,7 @@ +ioda2nc +*.o +*.a +.deps +.dirstamp +Makefile +Makefile.in diff --git a/met/src/tools/other/ioda2nc/Makefile.am b/met/src/tools/other/ioda2nc/Makefile.am new file mode 100644 index 0000000000..e4067043df --- /dev/null +++ b/met/src/tools/other/ioda2nc/Makefile.am @@ -0,0 +1,46 @@ +## @start 1 +## Makefile.am -- Process this file with automake to produce Makefile.in +## @end 1 + +MAINTAINERCLEANFILES = Makefile.in + +# Include the project definitions + +include ${top_srcdir}/Make-include + +# The program + +bin_PROGRAMS = ioda2nc +ioda2nc_SOURCES = ioda2nc.cc \ + ioda2nc_conf_info.cc ioda2nc_conf_info.h +ioda2nc_CPPFLAGS = ${MET_CPPFLAGS} +ioda2nc_LDFLAGS = ${MET_LDFLAGS} +ioda2nc_LDADD = -lvx_stat_out \ + -lvx_statistics \ + -lvx_shapedata \ + -lvx_gsl_prob \ + -lvx_analysis_util \ + -lvx_data2d_factory \ + -lvx_data2d_nc_met \ + -lvx_data2d_grib $(GRIB2_LIBS) \ + -lvx_data2d_nc_pinterp \ + -lvx_data2d_nccf \ + $(PYTHON_LIBS) \ + -lvx_data2d \ + -lvx_nc_obs \ + -lvx_nc_util \ + -lvx_regrid \ + -lvx_grid \ + -lvx_config \ + -lvx_gsl_prob \ + -lvx_pb_util \ + -lvx_cal \ + -lvx_util \ + $(PYTHON_LIBS) \ + -lvx_math \ + -lvx_color \ + -lvx_log \ + -lvx_summary \ + -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas $(BLIB_NAME) \ + $(FLIBS) + diff --git a/met/src/tools/other/ioda2nc/ioda2nc.cc b/met/src/tools/other/ioda2nc/ioda2nc.cc new file mode 100644 index 0000000000..2c7cbba7de --- /dev/null +++ b/met/src/tools/other/ioda2nc/ioda2nc.cc @@ -0,0 +1,1411 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2020 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// +// +// Filename: ioda2nc.cc +// +// Description: +// Based on user specified options, this tool filters point +// observations from a IODA input file, derives requested +// observation types, and writes the output to a NetCDF file. +// IODA observations must be reformatted in this way prior to +// using them in the Point-Stat tool. +// +// Mod# Date Name Description +// ---- ---- ---- ----------- +// 000 07-21-20 Howard Soh New +// +//////////////////////////////////////////////////////////////////////// + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apply_mask.h" +#include "ioda2nc_conf_info.h" +#include "vx_log.h" +#include "vx_nc_util.h" +#include "vx_util.h" +#include "vx_cal.h" +#include "vx_math.h" +#include "write_netcdf.h" + +#include "vx_summary.h" +#include "nc_obs_util.h" +#include "nc_summary.h" + +//////////////////////////////////////////////////////////////////////// + +// +// Constants +// + +static const char * DEF_CONFIG_NAME = "MET_BASE/config/IODA2NCConfig_default"; + +static const char *program_name = "ioda2nc"; + +//////////////////////////////////////////////////////////////////////// + +// +// Variables for command line arguments +// + +// StringArray to store IODA file name +static StringArray ioda_files; + +static StringArray core_dims; +static StringArray core_vars; + +// Output NetCDF file name +static ConcatString ncfile; + +// Input configuration file +static ConcatString config_file; +static IODA2NCConfInfo conf_info; +static NcObsOutputData nc_out_data; + +static Grid mask_grid; +static MaskPlane mask_area; +static MaskPoly mask_poly; +static StringArray mask_sid; + +// Beginning and ending retention times +static unixtime valid_beg_ut, valid_end_ut; + +// Number of IODA messages to process from the command line +static int nmsg = -1; +static int nmsg_percent = -1; + +static bool do_all_vars = false; +static StringArray obs_var_names; +static StringArray obs_var_units; +static StringArray obs_var_descs; + +static int compress_level = -1; + +//////////////////////////////////////////////////////////////////////// + +static IntArray filtered_times; +//static map variableTypeMap; + +static bool do_summary; +static bool save_summary_only = false; +static SummaryObs *summary_obs; +static NetcdfObsVars obs_vars; + + +//////////////////////////////////////////////////////////////////////// + +static int n_total_obs; // Running total of observations +static vector observations; + +// +// Output NetCDF file, dimensions, and variables +// +static NcFile *f_out = (NcFile *) 0; + +//////////////////////////////////////////////////////////////////////// + +static void initialize(); +static void process_command_line(int, char **); +static void open_netcdf(); +static void process_ioda_file(int); +static void write_netcdf_hdr_data(); +static void clean_up(); + +static void addObservation(const float *obs_arr, const ConcatString &hdr_typ, + const ConcatString &hdr_sid, const time_t hdr_vld, + const float hdr_lat, const float hdr_lon, const float hdr_elv, + const float quality_mark, const int buf_size); + +static bool keep_message_type(const char *); +static bool keep_station_id(const char *); +static bool keep_valid_time(const unixtime, const unixtime, const unixtime); + +static void usage(); +static void set_compress(const StringArray &); +static void set_config(const StringArray &); +static void set_ioda_files(const StringArray &); +static void set_mask_grid(const StringArray &); +static void set_mask_poly(const StringArray &); +static void set_mask_sid(const StringArray &); +static void set_logfile(const StringArray &); +static void set_nmsg(const StringArray &); +static void set_obs_var(const StringArray & a); +static void set_valid_beg_time(const StringArray &); +static void set_valid_end_time(const StringArray &); +static void set_verbosity(const StringArray &); + +static void cleanup_hdr_buf(char *hdr_buf, int buf_len); +static bool check_core_data(const bool, const bool, StringArray &, StringArray &); +static bool check_missing_thresh(float value); +static ConcatString find_meta_name(StringArray, StringArray); +static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, const int); +static bool get_meta_data_strings(NcFile *, const ConcatString, char *); +static bool get_obs_data_float(NcFile *, const ConcatString, + NcVar *, float *, int *, const int); +static bool has_postfix(std::string const &, std::string const &); + +//////////////////////////////////////////////////////////////////////// + + +int main(int argc, char *argv[]) { + int i; + + // Set handler to be called for memory allocation error + set_new_handler(oom); + + // Initialize static variables + initialize(); + + // Process the command line arguments + process_command_line(argc, argv); + + // Open the NetCDF file + open_netcdf(); + + // Process each IODA file + for(i=0; isummarizeObs(summaryInfo); + summary_obs->setSummaryInfo(summaryInfo); + } + + // Write the NetCDF file + write_netcdf_hdr_data(); + + // Deallocate memory and clean up + clean_up(); + + return(0); +} + +//////////////////////////////////////////////////////////////////////// + +void initialize() { + + n_total_obs = 0; + + nc_obs_initialize(); + + core_dims.clear(); + core_dims.add("nvars"); + core_dims.add("nlocs"); + //core_dims.add("nstring"); + //core_dims.add("ndatetime"); + + core_vars.clear(); + core_vars.add("datetime"); + core_vars.add("latitude"); + core_vars.add("longitude"); + + summary_obs = new SummaryObs(); + return; +} + +//////////////////////////////////////////////////////////////////////// + +void process_command_line(int argc, char **argv) { + CommandLine cline; + static const char *method_name = "process_command_line() -> "; + + // Check for zero arguments + if(argc == 1) usage(); + + // Initialize retention times + valid_beg_ut = valid_end_ut = (unixtime) 0; + + // Parse the command line into tokens + cline.set(argc, argv); + + // Set the usage function + cline.set_usage(usage); + + // Add the options function calls + cline.add(set_ioda_files, "-iodafile", 1); + cline.add(set_valid_beg_time, "-valid_beg", 1); + cline.add(set_valid_end_time, "-valid_end", 1); + cline.add(set_nmsg, "-nmsg", 1); + cline.add(set_obs_var, "-obs_var", 1); + cline.add(set_config, "-config", 1); + cline.add(set_mask_grid, "-mask_grid", 1); + cline.add(set_mask_poly, "-mask_poly", 1); + cline.add(set_mask_sid, "-mask_sid", 1); + cline.add(set_logfile, "-log", 1); + cline.add(set_verbosity, "-v", 1); + cline.add(set_compress, "-compress", 1); + + // Parse the command line + cline.parse(); + + // Check for error. There should be three arguments left: + // IODA, output NetCDF, and config filenames + if(cline.n() < 2) usage(); + + // Store the input file names + ioda_files.add(cline[0]); + if(cline.n() > 1) ncfile = cline[1]; + + // Create the default config file name + ConcatString default_config_file = replace_path(DEF_CONFIG_NAME); + + // List the config files and read the config files + if(0 < config_file.length()) { + if( !file_exists(config_file.c_str()) ) { + mlog << Error << "\n" << method_name + << "file does not exist \"" << config_file << "\"\n\n"; + exit(1); + } + mlog << Debug(1) + << "Default Config File: " << default_config_file << "\n" + << "User Config File: " << config_file << "\n"; + } + else mlog << Debug(1) + << "Default Config File: " << default_config_file << "\n"; + conf_info.read_config(default_config_file.c_str(), config_file.c_str()); + + // Process the configuration + conf_info.process_config(); + + // Check that valid_end_ut >= valid_beg_ut + if(valid_beg_ut != (unixtime) 0 && valid_end_ut != (unixtime) 0 + && valid_beg_ut > valid_end_ut) { + mlog << Error << "\nprocess_command_line() -> " + << "the ending time (" << unix_to_yyyymmdd_hhmmss(valid_end_ut) + << ") must be greater than the beginning time (" + << unix_to_yyyymmdd_hhmmss(valid_beg_ut) << ").\n\n"; + exit(1); + } + + do_summary = conf_info.getSummaryInfo().flag; + if(do_summary) save_summary_only = !conf_info.getSummaryInfo().raw_data; + else save_summary_only = false; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void open_netcdf() { + + // Create the output netCDF file for writing + mlog << Debug(1) << "Creating NetCDF File:\t\t" << ncfile << "\n"; + f_out = open_ncfile(ncfile.c_str(), true); + + // Check for a valid file + if(IS_INVALID_NC_P(f_out)) { + mlog << Error << "\nopen_netcdf() -> " + << "trouble opening output file: " << ncfile << "\n\n"; + + delete f_out; + f_out = (NcFile *) 0; + + exit(1); + } + + // Define netCDF variables + init_nc_dims_vars_config(obs_vars); + + // Add global attributes + write_netcdf_global(f_out, ncfile.text(), program_name); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void process_ioda_file(int i_pb) { + int npbmsg, npbmsg_total; + int idx, i_msg, i_read, n_file_obs, i_ret, n_hdr_obs; + int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly; + int rej_elv, rej_nobs; + double x, y; + + unixtime file_ut; + unixtime adjusted_file_ut; + unixtime msg_ut, beg_ut, end_ut; + unixtime min_msg_ut, max_msg_ut; + + ConcatString file_name, blk_prefix, blk_file, log_message; + ConcatString prefix; + char time_str[max_str_len]; + ConcatString start_time_str, end_time_str; + char min_time_str[max_str_len], max_time_str[max_str_len]; + + char hdr_typ[max_str_len]; + ConcatString hdr_sid; + char modified_hdr_typ[max_str_len]; + double hdr_lat, hdr_lon, hdr_elv; + unixtime hdr_vld_ut; + float obs_arr[OBS_ARRAY_LEN]; + + const int debug_level_for_performance = 3; + int start_t, end_t, method_start, method_end; + start_t = end_t = method_start = method_end = clock(); + + IntArray diff_file_times; + int diff_file_time_count; + StringArray variables_big_nlevels; + static const char *method_name = "process_ioda_file() "; + + bool apply_grid_mask = (conf_info.grid_mask.nx() > 0 && + conf_info.grid_mask.ny() > 0); + bool apply_area_mask = (conf_info.area_mask.nx() > 0 && + conf_info.area_mask.ny() > 0); + bool apply_poly_mask = (conf_info.poly_mask.n_points() > 0); + + hdr_typ[0] = 0; + + file_ut = beg_ut = end_ut = hdr_vld_ut = (unixtime) 0; + + // List the IODA file being processed + mlog << Debug(1) << "Processing IODA File:\t" << ioda_files[i_pb]<< "\n"; + + NcFile *f_in = open_ncfile(ioda_files[i_pb].c_str()); + + // Check for a valid file + if(IS_INVALID_NC_P(f_in)) { + mlog << Error << "\n" << method_name << "-> " + << "can't open input NetCDF file \"" << ioda_files[i_pb] + << "\" for reading.\n\n"; + delete f_in; + f_in = (NcFile *) 0; + + exit(1); + } + + // Initialize + filtered_times.clear(); + min_msg_ut = max_msg_ut = (unixtime) 0; + min_time_str[0] = NULL; + max_time_str[0] = NULL; + + // Set the file name for the IODA file + file_name << ioda_files[i_pb]; + + + int nrecs = 0; + //int nstring = 0; + StringArray var_names, dim_names; + StringArray metadata_vars; + StringArray obs_value_vars; + get_var_names(f_in, &var_names); + get_dim_names(f_in, &dim_names); + for(idx=0; idx= 6) { + for(idx=0; idx " + << "Please check the IODA file (required dimensions or meta variables are missing).\n\n"; + delete f_in; + f_in = (NcFile *) 0; + exit(1); + } + + // Compute the number of IODA records in the current file. + bool error_out = true; + int nvars = get_dim_value(f_in, "nvars", error_out); // number of variables + int nlocs = get_dim_value(f_in, "nlocs", error_out); // number of locations + int nstring = get_dim_value(f_in, "nstring", error_out); + + if(dim_names.has("nrecs")) nrecs = get_dim_value(f_in, "nrecs", false); + else { + nrecs = nvars * nlocs; + mlog << Debug(3) << "\n" << method_name << "-> " + << "nrecs dimension does not exist, so computed\n"; + } + NcVar in_hdr_vld_var = get_var(f_in, "datetime@MetaData"); + NcVar in_hdr_lat_var = get_var(f_in, "latitude@MetaData"); + NcVar in_hdr_lon_var = get_var(f_in, "longitude@MetaData"); + + int ndatetime; + if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); + else { + NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); + ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; + mlog << Debug(3) << "\n" << method_name << "-> " + << "ndatetime dimension does not exist!\n"; + } + mlog << Debug(5) << method_name << "-> dimensions: nvars=" << nvars << ", nlocs=" << nlocs + << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; + + npbmsg_total = npbmsg = nlocs; + + // Use the number of records requested by the user if there + // are enough present. + if(nmsg > 0 && nmsg < npbmsg) { + npbmsg = (nmsg_percent > 0 && nmsg_percent <= 100) + ? (npbmsg * nmsg_percent / 100) : nmsg; + } + + long lengths[2] = { nlocs, ndatetime }; + long offsets[2] = { 0, 0 }; + float *hdr_lat_arr = new float[nlocs]; + float *hdr_lon_arr = new float[nlocs]; + float *hdr_elv_arr = new float[nlocs]; + float *obs_pres_arr = new float[nlocs]; + float *obs_hght_arr = new float[nlocs]; + char *hdr_vld_block = new char[nlocs*ndatetime]; + char *hdr_msg_types = 0; + char *hdr_station_ids = 0; + vector v_qc_data; + vector v_obs_data; + + get_meta_data_float(f_in, metadata_vars, "pressure", obs_pres_arr, nlocs); + get_meta_data_float(f_in, metadata_vars, "height", obs_hght_arr, nlocs); + get_meta_data_float(f_in, metadata_vars, "elevation", hdr_elv_arr, nlocs); + + if(has_msg_type) { + hdr_msg_types = new char[nlocs*nstring]; + get_meta_data_strings(f_in, msg_type_name, hdr_msg_types); + } + + if(has_station_id) { + hdr_station_ids = new char[nlocs*nstring]; + get_meta_data_strings(f_in, station_id_name, hdr_station_ids); + } + + if(!get_nc_data(&in_hdr_lat_var, hdr_lat_arr, nlocs)) { + mlog << Error << "\n" << method_name << " -> " + << "trouble getting latitude\n\n"; + exit(1); + } + if(!get_nc_data(&in_hdr_lon_var, hdr_lon_arr, nlocs)) { + mlog << Error << "\n" << method_name << " -> " + << "trouble getting longitude\n\n"; + exit(1); + } + if(!get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets)) { + mlog << Error << "\n" << method_name << " -> " + << "trouble getting datetime\n\n"; + exit(1); + } + + StringArray raw_var_names; + if(do_all_vars || obs_var_names.n() == 0) raw_var_names = obs_value_vars; + else raw_var_names = obs_var_names; + if(obs_var_names.n() > 0) obs_var_names.clear(); + + NcVar obs_var, qc_var; + ConcatString unit_attr; + ConcatString desc_attr; + map name_map = conf_info.getObsVarMap(); + for(idx=0; idx " + << "processing \"" << obs_var_name << "\" variable!\n"; + obs_var = get_var(f_in, obs_var_name.c_str()); + v_qc_data.push_back(qc_data); + v_obs_data.push_back(obs_data); + unit_attr.clear(); + desc_attr.clear(); + get_obs_data_float(f_in, raw_var_names[idx], &obs_var, obs_data, qc_data, nlocs); + if(IS_VALID_NC(obs_var)) { + get_var_units(&obs_var, unit_attr); + get_att_value_string(&obs_var, "long_name", desc_attr); + } + obs_var_units.add(unit_attr); + obs_var_descs.add(desc_attr); + + // Replace the input variable name to the output variable name + ConcatString new_name = name_map[raw_var_names[idx]]; + if (0 < new_name.length()) obs_var_names.add(new_name); + else obs_var_names.add(raw_var_names[idx]); + } + + // Initialize counts + i_ret = n_file_obs = i_msg = 0; + rej_typ = rej_sid = rej_vld = rej_grid = rej_poly = 0; + rej_elv = rej_nobs = 0; + + bool showed_progress = false; + if(mlog.verbosity_level() >= debug_level_for_performance) { + end_t = clock(); + mlog << Debug(debug_level_for_performance) << " PERF: " << method_name << " " + << (end_t-start_t)/double(CLOCKS_PER_SEC) + << " seconds for preparing\n"; + start_t = clock(); + } + + log_message.add(" IODA messages"); + if(npbmsg != npbmsg_total) { + log_message << " (out of " << unixtime_to_string(npbmsg_total) << ")"; + } + mlog << Debug(2) << "Processing " << npbmsg << log_message << "...\n"; + + int bin_count = nint(npbmsg/20.0); + unixtime prev_hdr_vld_ut = (unixtime) 0; + map message_type_map = conf_info.getMessageTypeMap(); + + // Initialize + diff_file_time_count = 0; + + for(int idx=0; idx 0) { + if(bin_count > 0 && (i_read+1)%bin_count == 0) { + cout << nint((double) (i_read+1)/npbmsg*100.0) << "% " << flush; + showed_progress = true; + if(mlog.verbosity_level() >= debug_level_for_performance) { + end_t = clock(); + cout << (end_t-start_t)/double(CLOCKS_PER_SEC) + << " seconds\n"; + start_t = clock(); + } + } + } + + char valid_time[ndatetime+1]; + strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), ndatetime); + valid_time[ndatetime] = 0; + msg_ut = yyyymmddThhmmss_to_unix(valid_time); + + // Check to make sure that the message time hasn't changed + // from one IODA message to the next + if(file_ut == (unixtime) 0) { + adjusted_file_ut = file_ut = msg_ut; + + mlog << Debug(2) + << " IODA Time Center:\t\t" << unix_to_yyyymmdd_hhmmss(adjusted_file_ut) + << "\n"; + + // Check if valid_beg_ut and valid_end_ut were set on the + // command line. If so, use them. If not, use beg_ds and + // end_ds. + if(valid_beg_ut != (unixtime) 0 || + valid_end_ut != (unixtime) 0) { + beg_ut = valid_beg_ut; + end_ut = valid_end_ut; + } + else { + beg_ut = adjusted_file_ut + conf_info.beg_ds; + end_ut = adjusted_file_ut + conf_info.end_ds; + } + + if(beg_ut != (unixtime) 0) { + unix_to_yyyymmdd_hhmmss(beg_ut, start_time_str); + } + else { + start_time_str = "NO_BEG_TIME"; + } + + if(end_ut != (unixtime) 0) { + unix_to_yyyymmdd_hhmmss(end_ut, end_time_str); + } + else { + end_time_str = "NO_END_TIME"; + } + + mlog << Debug(2) << "Searching Time Window:\t\t" << start_time_str + << " to " << end_time_str << "\n"; + } + else if(file_ut != msg_ut) { + diff_file_time_count++; + if(!diff_file_times.has(msg_ut)) diff_file_times.add(msg_ut); + } + + if(has_msg_type) { + int buf_len = sizeof(modified_hdr_typ); + strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring); + hdr_typ[nstring] = NULL; + // Null terminate the message type string + cleanup_hdr_buf(hdr_typ, nstring); + + // If the message type is not listed in the configuration + // file and it is not the case that all message types should be + // retained, continue to the next IODA message + if(!keep_message_type(hdr_typ)) { + rej_typ++; + continue; + } + + if(0 < message_type_map.count((string)hdr_typ)) { + ConcatString mappedMessageType = message_type_map[(string)hdr_typ]; + mlog << Debug(6) << "\n" << method_name << " -> " + << "Switching report type \"" << hdr_typ + << "\" to message type \"" << mappedMessageType << "\".\n"; + if(mappedMessageType.length() < HEADER_STR_LEN) buf_len = HEADER_STR_LEN; + strncpy(modified_hdr_typ, mappedMessageType.c_str(), buf_len); + } + else { + strncpy(modified_hdr_typ, hdr_typ, buf_len); + } + modified_hdr_typ[buf_len-1] = NULL; + } + + if(has_station_id) { + char tmp_sid[nstring+1]; + strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring); + tmp_sid[nstring] = NULL; + cleanup_hdr_buf(tmp_sid, nstring); + hdr_sid = tmp_sid; + } + else hdr_sid.clear(); + + // If the station id is not listed in the configuration + // file and it is not the case that all station ids should be + // retained, continue to the next IODA message + if(!keep_station_id(hdr_sid.c_str())) { + rej_sid++; + continue; + } + + // Read the header array elements which consists of: + // LON LAT DHR ELV TYP T29 ITP + + // Longitude + hdr_lon = hdr_lon_arr[i_read]; + + // Latitude + hdr_lat = hdr_lat_arr[i_read]; + + // Elevation + hdr_elv = hdr_elv_arr[i_read]; + + // Compute the valid time and check if it is within the + // specified valid range + //hdr_vld_ut = msg_ut + (unixtime)nint(hdr[3]*sec_per_hour); + hdr_vld_ut = msg_ut; + if(0 == min_msg_ut || min_msg_ut > hdr_vld_ut) min_msg_ut = hdr_vld_ut; + if(max_msg_ut < hdr_vld_ut) max_msg_ut = hdr_vld_ut; + if(!keep_valid_time(hdr_vld_ut, beg_ut, end_ut)) { + if(!filtered_times.has(hdr_vld_ut, false)) { + filtered_times.add(hdr_vld_ut); + } + rej_vld++; + continue; + } + + // Rescale the longitude value from 0 to 360 -> -180 to 180 + hdr_lon = rescale_lon(hdr_lon); + + // If the lat/lon for the IODA message is not on the + // grid_mask, continue to the next IODA message + if(apply_grid_mask) { + conf_info.grid_mask.latlon_to_xy(hdr_lat, (-1.0*hdr_lon), x, y); + if(x < 0 || x >= conf_info.grid_mask.nx() || + y < 0 || y >= conf_info.grid_mask.ny()) { + rej_grid++; + continue; + } + + // Include the area mask rejection counts with the polyline since + // it is specified using the mask.poly config option. + if(apply_area_mask) { + if(!conf_info.area_mask.s_is_on(nint(x), nint(y))) { + rej_poly++; + continue; + } + } + } + + // If the lat/lon for the IODA message is not inside the mask + // polyline continue to the next IODA message. Multiply by + // -1 to convert from degrees_east to degrees_west + if(apply_poly_mask && + !conf_info.poly_mask.latlon_is_inside_dege(hdr_lat, hdr_lon)) { + rej_poly++; + continue; + } + + // Check if the message elevation is within the specified range. + // Missing data values for elevation are retained. + if(!check_missing_thresh(hdr_elv) && + (hdr_elv < conf_info.beg_elev || hdr_elv > conf_info.end_elev) ) { + rej_elv++; + continue; + } + + // Store the index to the header data + obs_arr[0] = (float) get_nc_hdr_cur_index(); + + n_hdr_obs = 0; + for(idx=0; idx 0) { + add_nc_header_to_array(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut, + hdr_lat, hdr_lon, hdr_elv); + i_msg++; + } + else { + rej_nobs++; + } + } // end for i_read + + if(showed_progress) { + log_message = "100% "; + if(mlog.verbosity_level() >= debug_level_for_performance) { + end_t = clock(); + log_message << (end_t-start_t)/double(CLOCKS_PER_SEC) << " seconds"; + } + cout << log_message << "\n"; + } + + int obs_buf_index = get_nc_obs_buf_index(); + if(obs_buf_index > 0) write_nc_obs_buffer(obs_buf_index); + + if(mlog.verbosity_level() > 0) cout << "\n" << flush; + + mlog << Debug(2) + << "Total records processed\t\t= " << npbmsg << "\n" + << "Rejected based on message type\t\t= " + << rej_typ << "\n" + << "Rejected based on station id\t\t= " + << rej_sid << "\n" + << "Rejected based on valid time\t\t= " + << rej_vld << "\n" + << "Rejected based on masking grid\t\t= " + << rej_grid << "\n" + << "Rejected based on masking polygon\t= " + << rej_poly << "\n" + << "Rejected based on elevation\t\t= " + << rej_elv << "\n" + << "Rejected based on zero observations\t= " + << rej_nobs << "\n" + << "Total Records retained\t\t= " + << i_msg << "\n" + << "Total observations retained or derived\t= " + << n_file_obs << "\n"; + + if(npbmsg == rej_vld && 0 < rej_vld) { + mlog << Warning << "\n" << method_name << " -> " + << "All records were filtered out by valid time.\n" + << "\tPlease adjust time range with \"-valid_beg\" and \"-valid_end\".\n" + << "\tmin/max obs time from IODA file: " << min_time_str + << " and " << max_time_str << ".\n" + << "\ttime range: " << start_time_str << " and " << end_time_str << ".\n"; + } + else { + mlog << Debug(1) << "Obs time between " << unix_to_yyyymmdd_hhmmss(min_msg_ut) + << " and " << unix_to_yyyymmdd_hhmmss(max_msg_ut) << "\n"; + + int debug_level = 5; + if(mlog.verbosity_level() >= debug_level) { + log_message = "Filtered time:"; + for(int kk=0; kk= debug_level_for_performance) { + method_end = clock(); + cout << " PERF: " << method_name << " " + << (method_end-method_start)/double(CLOCKS_PER_SEC) + << " seconds\n"; + } + + if(i_msg <= 0) { + mlog << Warning << "\n" << method_name << " -> " + << "No IODA records retained from file: " + << ioda_files[i_pb] << "\n\n"; + return; + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void write_netcdf_hdr_data() { + static const string method_name = "\nwrite_netcdf_hdr_data()"; + + const long hdr_count = (long) get_nc_hdr_cur_index(); + int deflate_level = compress_level; + if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression(); + + nc_out_data.processed_hdr_cnt = hdr_count; + nc_out_data.deflate_level = deflate_level; + nc_out_data.observations = observations; + nc_out_data.summary_obs = summary_obs; + nc_out_data.summary_info = conf_info.getSummaryInfo(); + + init_netcdf_output(f_out, obs_vars, nc_out_data, program_name); + + // Check for no messages retained + if(obs_vars.hdr_cnt <= 0) { + mlog << Error << method_name << " -> " + << "No IODA reocrds retained. Nothing to write.\n\n"; + // Delete the NetCDF file + remove_temp_file(ncfile); + exit(1); + } + + // Make sure all obs data is processed before handling header + write_observations(f_out, obs_vars, nc_out_data); + + StringArray nc_var_name_arr; + StringArray nc_var_unit_arr; + StringArray nc_var_desc_arr; + const long var_count = obs_var_names.n(); + const long units_count = obs_var_units.n(); + + for(int i=0; iaddObservationObj(obs); + return; +} + +//////////////////////////////////////////////////////////////////////// + +void clean_up() { + + if(f_out) { + delete f_out; + f_out = (NcFile *) 0; + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void cleanup_hdr_buf(char *hdr_buf, int buf_len) { + int i; + hdr_buf[buf_len] = '\0'; + // Change the trailing blank space to a null + for(i=buf_len-1; i>=0; i--) { + if(' ' == hdr_buf[i]) { + hdr_buf[i] = '\0'; + if(i > 0 && ' ' != hdr_buf[i-1]) break; + } + } +} + +//////////////////////////////////////////////////////////////////////// + +bool keep_message_type(const char *mt_str) { + bool keep = false; + + keep = conf_info.message_type.n_elements() == 0 || + conf_info.message_type.has(mt_str, false); + + return(keep); +} + +//////////////////////////////////////////////////////////////////////// + +bool keep_station_id(const char *sid_str) { + + return(conf_info.station_id.n_elements() == 0 || + conf_info.station_id.has(sid_str, false)); +} + +//////////////////////////////////////////////////////////////////////// + +bool keep_valid_time(const unixtime ut, + const unixtime min_ut, const unixtime max_ut) { + bool keep = true; + + // If min_ut and max_ut both set, check the range + if(min_ut != (unixtime) 0 && max_ut != (unixtime) 0) { + if(ut < min_ut || ut > max_ut) keep = false; + } + // If only min_ut set, check the lower bound + else if(min_ut != (unixtime) 0 && max_ut == (unixtime) 0) { + if(ut < min_ut) keep = false; + } + // If only max_ut set, check the upper bound + else if(min_ut == (unixtime) 0 && max_ut != (unixtime) 0) { + if(ut > max_ut) keep = false; + } + + return(keep); +} + +//////////////////////////////////////////////////////////////////////// + +bool check_core_data(const bool has_msg_type, const bool has_station_id, + StringArray &dim_names, StringArray &metadata_vars) { + bool is_netcdf_ready = true; + static const char *method_name = "check_core_data() -> "; + + for(int idx=0; idx " + << "core dimension \"" << core_dims[idx] << "\" is missing.\n\n"; + is_netcdf_ready = false; + } + } + + if(has_msg_type || has_station_id) { + if(!dim_names.has("nstring")) { + mlog << Error << "\n" << method_name << "-> " + << "core dimension \"nstring\" is missing.\n\n"; + is_netcdf_ready = false; + } + } + + for(int idx=0; idx " + << "core variable \"" << core_vars[idx] << "\" is missing.\n\n"; + is_netcdf_ready = false; + } + } + return is_netcdf_ready; +} + +//////////////////////////////////////////////////////////////////////// + +bool check_missing_thresh(float value) { + bool check = false; + for(int idx=0; idx 0) { + ConcatString ioda_name = metadata_name; + ioda_name.add("@MetaData"); + NcVar meta_var = get_var(f_in, ioda_name.c_str()); + if(IS_VALID_NC(meta_var)) { + status = get_nc_data(&meta_var, metadata_buf, nlocs); + if(!status) mlog << Debug(3) << method_name + << "trouble getting " << metadata_name << "\n"; + } + } + else mlog << Warning << "\n" << method_name + << "Metadata for " << metadata_key << " does not exist!\n\n"; + if(status) { + for(int idx=0; idx " + << "trouble getting " << metadata_name << "\n\n"; + exit(1); + } + } + return status; +} + +//////////////////////////////////////////////////////////////////////// + +bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, + NcVar *obs_var, float *obs_buf, int *qc_buf, + const int nlocs) { + bool status = false; + static const char *method_name = "get_obs_data_float() -> "; + + if(IS_VALID_NC_P(obs_var)) { + status = get_nc_data(obs_var, obs_buf, nlocs); + if(status) { + for(int idx=0; idx 0) { + ConcatString ioda_name = var_name; + ioda_name.add("@PreQC"); + NcVar qc_var = get_var(f_in, ioda_name.c_str()); + if(IS_VALID_NC(qc_var)) { + status = get_nc_data(&qc_var, qc_buf, nlocs); + if(!status) mlog << Warning << "\n" << method_name + << "trouble getting " << ioda_name << "\n\n"; + } + else mlog << Warning << "\n" << method_name + << "\"" << ioda_name << "\" does not exist!\n\n"; + } + if(status) { + for(int idx=0; idx= postfix.length()) { + return (0 == str_buf.compare(str_buf.length() - postfix.length(), postfix.length(), postfix)); + } else { + return false; + } +} + +//////////////////////////////////////////////////////////////////////// + +void usage() { + + cout << "\n*** Model Evaluation Tools (MET" << met_version + << ") ***\n\n" + + << "Usage: " << program_name << "\n" + << "\tioda_file\n" + << "\tnetcdf_file\n" + << "\t[-config config_file]\n" + << "\t[-obs_var var]\n" + << "\t[-iodafile ioda_file]\n" + << "\t[-valid_beg time]\n" + << "\t[-valid_end time]\n" + << "\t[-nmsg n]\n" + << "\t[-log file]\n" + << "\t[-v level]\n" + << "\t[-compress level]\n\n" + + << "\twhere\t\"ioda_file\" is the input IODA " + << "observation file to be converted to netCDF format " + << "(required).\n" + + << "\t\t\"netcdf_file\" indicates the name of the output " + << "netCDF file to be written (required).\n" + + << "\t\t\"-config config_file\" is a IODA2NCConfig file containing the " + << "desired configuration settings (optional).\n" + + << "\t\t\"-obs_var var1,...\" or multiple \"-obs_var var\" sets the variable list to be saved" + << " from input IODA observation files (optional, default: all).\n" + + << "\t\t\"-iodafile ioda_file\" may be used to specify " + << "additional input IODA observation files to be used " + << "(optional).\n" + + << "\t\t\"-valid_beg time\" in YYYYMMDD[_HH[MMSS]] sets the " + << "beginning of the retention time window (optional).\n" + + << "\t\t\"-valid_end time\" in YYYYMMDD[_HH[MMSS]] sets the " + << "end of the retention time window (optional).\n" + + << "\t\t\"-nmsg n\" indicates the number of IODA records " + << "to process (optional).\n" + + << "\t\t\"-log file\" outputs log messages to the specified " + << "file (optional).\n" + + << "\t\t\"-v level\" overrides the default level of logging (" + << mlog.verbosity_level() << ") (optional).\n" + + << "\t\t\"-compress level\" overrides the compression level of NetCDF variable (" + << conf_info.conf.nc_compression() << ") (optional).\n\n" + + << flush; + + exit(1); +} + +//////////////////////////////////////////////////////////////////////// + +void set_ioda_files(const StringArray & a) +{ + ioda_files.add(a[0]); +} + +//////////////////////////////////////////////////////////////////////// + +void set_valid_beg_time(const StringArray & a) +{ + valid_beg_ut = timestring_to_unix(a[0].c_str()); +} + +//////////////////////////////////////////////////////////////////////// + +void set_valid_end_time(const StringArray & a) +{ + valid_end_ut = timestring_to_unix(a[0].c_str()); +} + +//////////////////////////////////////////////////////////////////////// + +void set_nmsg(const StringArray & a) +{ + nmsg = atoi(a[0].c_str()); + int tmp_len = a[0].length(); + if(1 < tmp_len) { + if(a[0][tmp_len-1] == '%') { + nmsg_percent = nmsg; + } + } +} + +//////////////////////////////////////////////////////////////////////// + +void set_config(const StringArray & a) { + config_file = a[0]; +} + +//////////////////////////////////////////////////////////////////////// + +void set_mask_grid(const StringArray & a) { + + // List the grid masking file + mlog << Debug(1) + << "Grid Masking: " << a[0] << "\n"; + + parse_grid_mask(a[0], mask_grid); + + // List the grid mask + mlog << Debug(2) + << "Parsed Masking Grid: " << mask_grid.name() << " (" + << mask_grid.nx() << " x " << mask_grid.ny() << ")\n"; +} + +//////////////////////////////////////////////////////////////////////// + +void set_mask_poly(const StringArray & a) { + ConcatString mask_name; + + // List the poly masking file + mlog << Debug(1) + << "Polyline Masking File: " << a[0] << "\n"; + + parse_poly_mask(a[0], mask_poly, mask_grid, mask_area, mask_name); + + // List the mask information + if(mask_poly.n_points() > 0) { + mlog << Debug(2) + << "Parsed Masking Polyline: " << mask_poly.name() + << " containing " << mask_poly.n_points() << " points\n"; + } + + // List the area mask information + if(mask_area.nx() > 0 || mask_area.ny() > 0) { + mlog << Debug(2) + << "Parsed Masking Area: " << mask_name + << " for (" << mask_grid.nx() << " x " << mask_grid.ny() + << ") grid\n"; + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void set_mask_sid(const StringArray & a) { + ConcatString mask_name; + + // List the station ID mask + mlog << Debug(1) << "Station ID Mask: " << a[0] << "\n"; + + parse_sid_mask(a[0], mask_sid, mask_name); + + // List the length of the station ID mask + mlog << Debug(2) << "Parsed Station ID Mask: " << mask_name + << " containing " << mask_sid.n_elements() << " points\n"; +} + +//////////////////////////////////////////////////////////////////////// + +void set_obs_var(const StringArray & a) +{ + if("_all_" == a[0] || "all" == a[0]) do_all_vars = true; + else { + ConcatString arg = a[0]; + obs_var_names.add(arg.split(",+ ")); + } +} + +//////////////////////////////////////////////////////////////////////// + +void set_logfile(const StringArray & a) +{ + ConcatString filename; + + filename = a[0]; + + mlog.open_log_file(filename); +} + +//////////////////////////////////////////////////////////////////////// + +void set_verbosity(const StringArray & a) +{ + mlog.set_verbosity_level(atoi(a[0].c_str())); +} + +//////////////////////////////////////////////////////////////////////// + +void set_compress(const StringArray & a) { + compress_level = atoi(a[0].c_str()); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/ioda2nc/ioda2nc_conf_info.cc b/met/src/tools/other/ioda2nc/ioda2nc_conf_info.cc new file mode 100644 index 0000000000..0b849be48f --- /dev/null +++ b/met/src/tools/other/ioda2nc/ioda2nc_conf_info.cc @@ -0,0 +1,179 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2020 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include + +#include "ioda2nc_conf_info.h" + +#include "vx_data2d_factory.h" +#include "apply_mask.h" +#include "grib_strings.h" +#include "vx_log.h" + +//////////////////////////////////////////////////////////////////////// +// +// Code for class IODA2NCConfInfo +// +//////////////////////////////////////////////////////////////////////// + +IODA2NCConfInfo::IODA2NCConfInfo() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +IODA2NCConfInfo::~IODA2NCConfInfo() { + clear(); +} + +//////////////////////////////////////////////////////////////////////// + +void IODA2NCConfInfo::init_from_scratch() { + clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void IODA2NCConfInfo::clear() { + + // Initialize values + message_type.clear(); + station_id.clear(); + valid_beg_ut = valid_end_ut = 0; + beg_ds = end_ds = bad_data_int; + grid_mask.clear(); + area_mask.clear(); + poly_mask.clear(); + beg_elev = end_elev = bad_data_double; + beg_level = end_level = bad_data_double; + quality_mark_thresh = bad_data_int; + version.clear(); + obs_name_map.clear(); + message_type_map.clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void IODA2NCConfInfo::read_config(const char *default_file_name, + const char *user_file_name) { + + // Read the config file constants + conf.read(replace_path(config_const_filename).c_str()); + + // Read the default config file + conf.read(default_file_name); + + // Read the user-specified config file + if (user_file_name && strcmp(user_file_name, "")) conf.read(user_file_name); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void IODA2NCConfInfo::process_config() { + int i; + ConcatString s, mask_name; + StringArray sa; + StringArray * sid_list = 0; + Dictionary *dict = (Dictionary *) 0; + static const char *method_name = "IODA2NCConfInfo::process_config() -> "; + + // Dump the contents of the config file + if(mlog.verbosity_level() >= 5) conf.dump(cout); + + // Initialize + clear(); + + // Conf: version + version = parse_conf_version(&conf); + + // Conf: message_type + message_type = conf.lookup_string_array(conf_key_message_type); + + // Conf: message_type_group_map + map group_map; + group_map = parse_conf_message_type_group_map(&conf); + + // Expand the values for any message type group names + for(i=0; i 0) { + message_type.add(group_map[message_type[i]]); + } + } + + // Conf: station_id + sa = conf.lookup_string_array(conf_key_station_id); + sid_list = new StringArray [sa.n_elements()]; + for(i=0; i 15) { + mlog << Warning << "\nIODA2NCConfInfo::process_config() -> " + << "the \"" << conf_key_quality_mark_thresh + << "\" entry (" << quality_mark_thresh + << ") should be set between 0 and 15.\n\n"; + } + + // Conf: obs_name_map + obs_name_map = parse_conf_obs_name_map(&conf); + message_type_map = parse_conf_message_type_map(&conf); + metadata_map = parse_conf_metadata_map(&conf); + + if ( sid_list ) delete [] sid_list; + + return; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/ioda2nc/ioda2nc_conf_info.h b/met/src/tools/other/ioda2nc/ioda2nc_conf_info.h new file mode 100644 index 0000000000..8751ca6df7 --- /dev/null +++ b/met/src/tools/other/ioda2nc/ioda2nc_conf_info.h @@ -0,0 +1,80 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2020 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __IODA2NC_CONF_INFO_H__ +#define __IODA2NC_CONF_INFO_H__ + +//////////////////////////////////////////////////////////////////////// + +#include + +#include "vx_config.h" +#include "vx_analysis_util.h" +#include "vx_grid.h" +#include "vx_util.h" +#include "vx_cal.h" +#include "vx_math.h" + +//////////////////////////////////////////////////////////////////////// + +class IODA2NCConfInfo { + + private: + + void init_from_scratch(); + + public: + + // IODA2NC configuration object + MetConfig conf; + + // Store data parsed from the IODA2NC configuration object + StringArray message_type; // Obseration message type + StringArray station_id; // Observation location station id + int beg_ds; // Time range of observations to be retained, + int end_ds; // Defined relative to the PrepBufr center time (seconds) + Grid grid_mask; // Grid masking region + MaskPlane area_mask; // Data masking region + MaskPoly poly_mask; // Lat/Lon polyline masking region + unixtime valid_beg_ut; + unixtime valid_end_ut; + double beg_elev; // Range of observing location elevations to be retained + double end_elev; + double beg_level; // Range of level values to be retained + double end_level; + StringArray obs_var; // IODA variiable names + int quality_mark_thresh; // Quality marks to be retained + ThreshArray missing_thresh; // Fill value thresh array + ConcatString version; // Config file version + + map obs_name_map; + map message_type_map; + map metadata_map; + StringArray surface_message_types; + TimeSummaryInfo timeSummaryInfo; + + IODA2NCConfInfo(); + ~IODA2NCConfInfo(); + + void clear(); + + map getObsVarMap() const { return obs_name_map; } + map getMessageTypeMap() const { return message_type_map; } + TimeSummaryInfo getSummaryInfo() const { return timeSummaryInfo; }; + + void read_config(const char *, const char *); + void process_config(); +}; + +//////////////////////////////////////////////////////////////////////// + +#endif /* __IODA2NC_CONF_INFO_H__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index c5510f4cf5..2921d40e82 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -673,21 +673,21 @@ void get_variable_info(const char* tbl_filename) { if ('0' != line[BUFR_NUMBER_START]) continue; strncpy(var_name, (line+BUFR_NAME_START), BUFR_NAME_LEN); - var_name[BUFR_NAME_LEN] = bad_data_char; + var_name[BUFR_NAME_LEN] = NULL; for (int idx=(BUFR_NAME_LEN-1); idx >=0; idx--) { if (' ' != var_name[idx] ) break; - var_name[idx] = '\0'; + var_name[idx] = NULL; } if (0 == strlen(var_name)) continue; var_count1++; strncpy(var_desc, (line+BUFR_DESCRIPTION_START), BUFR_DESCRIPTION_LEN); - var_desc[BUFR_DESCRIPTION_LEN] = bad_data_char; + var_desc[BUFR_DESCRIPTION_LEN] = NULL; for (int idx=(BUFR_DESCRIPTION_LEN-1); idx>=0; idx--) { if (' ' != var_desc[idx] && '|' != var_desc[idx]) { break; } - var_desc[idx] = '\0'; + var_desc[idx] = NULL; } mlog << Debug(10) << method_name << " sec. 1 var: [" << var_name << "] \tdesc: [" << var_desc << "]\n"; @@ -701,20 +701,20 @@ void get_variable_info(const char* tbl_filename) { if (NULL == strstr(line,"EVENT")) continue; strncpy(var_name, (line+BUFR_NAME_START), BUFR_NAME_LEN); - var_name[BUFR_NAME_LEN] = bad_data_char; + var_name[BUFR_NAME_LEN] = NULL; for (int idx=(BUFR_NAME_LEN-1); idx >=0; idx--) { if (' ' != var_name[idx] ) break; - var_name[idx] = '\0'; + var_name[idx] = NULL; } //if (NULL == strstr(var_name,"EVENT")) continue; strncpy(var_desc, (line+BUFR_SEQUENCE_START), BUFR_SEQUENCE_LEN); - var_desc[BUFR_SEQUENCE_LEN] = bad_data_char; + var_desc[BUFR_SEQUENCE_LEN] = NULL; for (int idx=(BUFR_SEQUENCE_LEN-1); idx>=0; idx--) { if (' ' != var_desc[idx] && '|' != var_desc[idx]) { break; } - var_desc[idx] = '\0'; + var_desc[idx] = NULL; } mlog << Debug(10) << method_name << " event: [" << var_name << "] \tdesc: [" << var_desc << "]\n"; @@ -729,10 +729,10 @@ void get_variable_info(const char* tbl_filename) { if ('-' == line[BUFR_NAME_START]) break; strncpy(var_name, (line+BUFR_NAME_START), BUFR_NAME_LEN); - var_name[BUFR_NAME_LEN] = bad_data_char; + var_name[BUFR_NAME_LEN] = NULL; for (int idx=(BUFR_NAME_LEN-1); idx >=0; idx--) { if (' ' != var_name[idx] ) break; - var_name[idx] = '\0'; + var_name[idx] = NULL; } if (NULL != strstr(line,"CCITT IA5")) { @@ -741,12 +741,12 @@ void get_variable_info(const char* tbl_filename) { } else { strncpy(var_unit_str, (line+BUFR_UNIT_START), BUFR_UNIT_LEN); - var_unit_str[BUFR_UNIT_LEN] = bad_data_char; + var_unit_str[BUFR_UNIT_LEN] = NULL; for (int idx=(BUFR_UNIT_LEN-1); idx>=0; idx--) { if (' ' != var_unit_str[idx] && '|' != var_unit_str[idx]) { break; } - var_unit_str[idx] = '\0'; + var_unit_str[idx] = NULL; } var_names.add(var_name); var_units.add(var_unit_str); @@ -858,8 +858,8 @@ void process_pbfile(int i_pb) { // Initialize filtered_times.clear(); min_msg_ut = max_msg_ut = (unixtime) 0; - min_time_str[0] = bad_data_char; - max_time_str[0] = bad_data_char; + min_time_str[0] = NULL; + max_time_str[0] = NULL; // Set the file name for the PrepBufr file file_name << pbfile[i_pb]; @@ -1044,6 +1044,7 @@ void process_pbfile(int i_pb) { // Get the next PrepBufr message (header only) ireadns_(&unit, hdr_typ, &i_date); readpb_hdr_(&unit, &i_ret, hdr); + hdr_typ[max_str_len-1] = 0; snprintf(time_str, sizeof(time_str), "%.10i", i_date); msg_ut = yyyymmddhh_to_unix(time_str); @@ -1276,6 +1277,7 @@ void process_pbfile(int i_pb) { // Get the next PrepBufr message readpb_(&unit, &i_ret, &nlev, hdr, evns, &nlev_max_req); + int max_buf = sizeof(modified_hdr_typ); // Special handling for "AIRNOW" and "ANOWPM" bool is_airnow = (0 == strcmp("AIRNOW", hdr_typ) || 0 == strcmp("ANOWPM", hdr_typ)); @@ -1285,16 +1287,14 @@ void process_pbfile(int i_pb) { mlog << Debug(6) << "\n" << method_name << " -> " << "Switching report type \"" << hdr_typ << "\" to message type \"" << mappedMessageType << "\".\n"; - if (mappedMessageType.length() < HEADER_STR_LEN) { - strncpy(modified_hdr_typ, mappedMessageType.c_str(), sizeof(modified_hdr_typ)); - } - else { - strncpy(modified_hdr_typ, mappedMessageType.c_str(), HEADER_STR_LEN); - } + if (mappedMessageType.length() > HEADER_STR_LEN) max_buf = HEADER_STR_LEN; + strncpy(modified_hdr_typ, mappedMessageType.c_str(), max_buf); } else { strncpy(modified_hdr_typ, hdr_typ, sizeof(modified_hdr_typ)); } + if (max_buf >= max_str_len) max_buf--; + modified_hdr_typ[max_buf] = NULL; // Search through the observation values and store them as: // HDR_ID GC LVL HGT OB @@ -3440,6 +3440,7 @@ void usage() { << "\t[-nmsg n]\n" << "\t[-index]\n" << "\t[-dump path]\n" + << "\t[-obs_var var]\n" << "\t[-log file]\n" << "\t[-v level]\n" << "\t[-compress level]\n\n" @@ -3471,9 +3472,12 @@ void usage() { << "\"prepbufr_file\" should also be dumped to text files " << "in the directory specified (optional).\n" + << "\t\t\"-obs_var var1,...\" or multiple \"-obs_var var\" sets the variable list to be saved" + << " from input BUFR files (optional, default: all).\n" + << "\t\t\"-index\" indicates that the meta data (available variables and headers)" << " is extracted from \"prepbufr_file\" (optional). " - << "No NetCDF outputs. \"-all\" or \"-vars\" is ignored.\n" + << "No NetCDF outputs. \"-obs_var\" is ignored.\n" << "\t\t\"-log file\" outputs log messages to the specified " << "file (optional).\n" diff --git a/scripts/fortify/run_fortify_sca.sh b/scripts/fortify/run_fortify_sca.sh index b95f4b0178..83335d4e1f 100755 --- a/scripts/fortify/run_fortify_sca.sh +++ b/scripts/fortify/run_fortify_sca.sh @@ -34,7 +34,7 @@ if [[ $# -lt 1 ]]; then usage; exit; fi # Check that FORTIFY_BIN is defined if [ -z ${FORTIFY_BIN+x} ]; then - echo "ERROR: ${FORTIFY_BIN} must be set" + echo "ERROR: FORTIFY_BIN must be set" exit 1 fi diff --git a/test/bin/unit_test.sh b/test/bin/unit_test.sh index 079c0315de..2b4bfac34b 100755 --- a/test/bin/unit_test.sh +++ b/test/bin/unit_test.sh @@ -72,6 +72,7 @@ UNIT_XML="unit_ascii2nc.xml \ unit_hira.xml \ unit_interp_shape.xml \ unit_lidar2nc.xml \ + unit_ioda2nc.xml \ unit_airnow.xml \ unit_python.xml \ unit_point2grid.xml \ diff --git a/test/config/IODA2NCConfig_mask b/test/config/IODA2NCConfig_mask new file mode 100644 index 0000000000..dea102d973 --- /dev/null +++ b/test/config/IODA2NCConfig_mask @@ -0,0 +1,122 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IODA2NC configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// IODA message type +// +message_type = [ ${MESSAGE_TYPE} ]; + +// +// Mapping of message type group name to comma-separated list of values +// Derive PRMSL only for SURFACE message types +// +//message_type_group_map = []; + +// +// Mapping of input IODA message types to output message types +// +message_type_map = []; + +// +// IODA station ID +// +station_id = [ ${STATION_ID} ]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation time window +// +obs_window = { + beg = -5400; + end = 5400; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation retention regions +// +mask = { + grid = "${MASK_GRID}"; + poly = "${MASK_POLY}"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observing location elevation +// +elevation_range = { + beg = -1000; + end = 100000; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation types +// +//////////////////////////////////////////////////////////////////////////////// + +// +// IODA variable names to retain or derive. +// Use obs_bufr_map to rename variables in the output. +// If empty or 'all', process all available variables. +// +//obs_var = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input IODA variable names to output variables names. +// The default IODA map, obs_var_map, is appended to this map. +// +obs_name_map = [ + { key = "wind_direction"; val = "WDIR"; }, + { key = "wind_speed"; val = "WIND"; } +]; + +// +// Default mapping for Metadata. +// +//metadata_map = []; + +//missing_thresh = []; + +//////////////////////////////////////////////////////////////////////////////// + +quality_mark_thresh = 0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Time periods for the summarization +// obs_var (string array) is added and works like grib_code (int array) +// when use_var_id is enabled and variable names are saved. +// +time_summary = { + flag = FALSE; + raw_data = FALSE; + beg = "000000"; + end = "235959"; + step = 300; + width = 600; + grib_code = []; + obs_var = [ "WIND" ]; + type = [ "min", "max", "range", "mean", "stdev", "median", "p80" ]; + vld_freq = 0; + vld_thresh = 0.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +version = "V10.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/IODA2NCConfig_summary b/test/config/IODA2NCConfig_summary new file mode 100644 index 0000000000..97b05393eb --- /dev/null +++ b/test/config/IODA2NCConfig_summary @@ -0,0 +1,122 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IODA2NC configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// IODA message type +// +//message_type = []; + +// +// Mapping of message type group name to comma-separated list of values +// Derive PRMSL only for SURFACE message types +// +//message_type_group_map = []; + +// +// Mapping of input IODA message types to output message types +// +//message_type_map = []; + +// +// IODA station ID +// +//station_id = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation time window +// +obs_window = { + beg = -5400; + end = 5400; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation retention regions +// +mask = { + grid = ""; + poly = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observing location elevation +// +elevation_range = { + beg = -1000; + end = 100000; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation types +// +//////////////////////////////////////////////////////////////////////////////// + +// +// IODA variable names to retain or derive. +// Use obs_bufr_map to rename variables in the output. +// If empty or 'all', process all available variables. +// +//obs_var = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input IODA variable names to output variables names. +// The default IODA map, obs_var_map, is appended to this map. +// +obs_name_map = [ + { key = "wind_direction"; val = "WDIR"; }, + { key = "wind_speed"; val = "WIND"; } +]; + +// +// Default mapping for Metadata. +// +//metadata_map = []; + +//missing_thresh = []; + +//////////////////////////////////////////////////////////////////////////////// + +quality_mark_thresh = 0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Time periods for the summarization +// obs_var (string array) is added and works like grib_code (int array) +// when use_var_id is enabled and variable names are saved. +// +time_summary = { + flag = TRUE; + raw_data = TRUE; + beg = "000000"; + end = "235959"; + step = 300; + width = 600; + grib_code = []; + obs_var = [ "WIND" ]; + type = [ "min", "max", "range", "mean", "stdev", "median", "p80" ]; + vld_freq = 0; + vld_thresh = 0.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +version = "V10.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_ioda2nc.xml b/test/xml/unit_ioda2nc.xml new file mode 100644 index 0000000000..0006da34d8 --- /dev/null +++ b/test/xml/unit_ioda2nc.xml @@ -0,0 +1,103 @@ + + + + + + + + + + + +]> + + + + &TEST_DIR; + true + + + + &MET_BIN;/ioda2nc + + STATION_ID "KEKA" + MASK_GRID + MASK_POLY + MESSAGE_TYPE + + \ + &DATA_DIR_OBS;/ioda/ioda.NC001007.2020031012.nc \ + &OUTPUT_DIR;/ioda2nc/ioda.NC001007.2020031012.mask_sid.nc \ + -config &CONFIG_DIR;/IODA2NCConfig_mask \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/ioda.NC001007.2020031012.mask_sid.nc + + + + + &MET_BIN;/ioda2nc + \ + &DATA_DIR_OBS;/ioda/odb_sonde_16019.nc4 \ + &OUTPUT_DIR;/ioda2nc/odb_sonde_16019_all.nc \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/odb_sonde_16019_all.nc + + + + + &MET_BIN;/ioda2nc + \ + &DATA_DIR_OBS;/ioda/ioda.NC001007.2020031012.nc \ + &OUTPUT_DIR;/ioda2nc/ioda.NC001007.2020031012.summary.nc \ + -config &CONFIG_DIR;/IODA2NCConfig_summary \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/ioda.NC001007.2020031012.summary.nc + + + +