From 1123b415dc5622450918b5815472f0dc4e6131dc Mon Sep 17 00:00:00 2001 From: hsoh-u Date: Sat, 29 Oct 2022 21:17:45 -0600 Subject: [PATCH] Feature #2068 ioda2nc v2.0 (#2307) Co-authored-by: Howard Soh Co-authored-by: John Halley Gotway Co-authored-by: MET Tools Test Account --- .github/jobs/set_job_controls.sh | 2 +- .../build_docker_and_trigger_metplus.yml | 2 +- data/config/IODA2NCConfig_default | 11 +- docs/Users_Guide/reformat_point.rst | 13 +- internal/scripts/docker/Dockerfile | 2 +- internal/scripts/docker/Dockerfile.copy | 2 +- .../scripts/environment/development.docker | 2 +- internal/test_unit/xml/unit_ioda2nc.xml | 38 ++ src/basic/vx_config/config_constants.h | 2 + src/basic/vx_config/config_util.cc | 7 + src/basic/vx_config/config_util.h | 11 +- .../vx_data2d_factory/is_netcdf_file.cc | 7 +- src/libcode/vx_nc_obs/nc_obs_util.cc | 35 +- src/libcode/vx_nc_util/nc_utils.cc | 340 +++++++++++++--- src/libcode/vx_nc_util/nc_utils.h | 38 +- src/libcode/vx_nc_util/nc_utils.hpp | 29 +- src/tools/other/ioda2nc/ioda2nc.cc | 380 +++++++++++++----- src/tools/other/ioda2nc/ioda2nc_conf_info.cc | 1 + src/tools/other/ioda2nc/ioda2nc_conf_info.h | 1 + 19 files changed, 704 insertions(+), 219 deletions(-) diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index b26f6d577d..e897aaa912 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -6,7 +6,7 @@ run_unit_tests=false run_diff=false run_update_truth=false met_base_repo=met-base -met_base_tag=v1.0 +met_base_tag=v1.1 input_data_version=develop truth_data_version=develop diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index e6c8c86a3f..15cd652caf 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -29,7 +29,7 @@ jobs: env: SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite MET_BASE_REPO: met-base - MET_BASE_TAG: v1.0 + MET_BASE_TAG: v1.1 - name: Push Docker Image run: .github/jobs/push_docker_image.sh diff --git a/data/config/IODA2NCConfig_default b/data/config/IODA2NCConfig_default index 2ac36c27ce..124854005e 100644 --- a/data/config/IODA2NCConfig_default +++ b/data/config/IODA2NCConfig_default @@ -92,7 +92,16 @@ metadata_map = [ { key = "station_id"; val = "station_id,report_identifier"; }, { key = "pressure"; val = "air_pressure,pressure"; }, { key = "height"; val = "height,height_above_mean_sea_level"; }, - { key = "elevation"; val = ""; } + { key = "datetime"; val = "datetime,dateTime"; }, + { key = "elevation"; val = "elevation,station_elevation"; } +]; + +// +// Default mapping for obs to qc. +// +obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } ]; missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst index f4430cd8e1..d1b0cabd6e 100644 --- a/docs/Users_Guide/reformat_point.rst +++ b/docs/Users_Guide/reformat_point.rst @@ -961,13 +961,24 @@ _____________________ { key = "station_id"; val = "station_id,report_identifier"; }, { key = "pressure"; val = "air_pressure,pressure"; }, { key = "height"; val = "height,height_above_mean_sea_level"; }, - { key = "elevation"; val = ""; } + { key = "elevation"; val = "elevation,station_elevation"; } ]; This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of metadata for IODA data files. _____________________ +.. code-block:: none + + obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } + ]; + +This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of QC variable name for IODA data files. + +_____________________ + .. code-block:: none missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index 4dafd977b3..8ec46e33db 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base -ARG MET_BASE_TAG=v1.0 +ARG MET_BASE_TAG=v1.1 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} MAINTAINER John Halley Gotway diff --git a/internal/scripts/docker/Dockerfile.copy b/internal/scripts/docker/Dockerfile.copy index 5c51408fdf..7d0fc6ae6f 100644 --- a/internal/scripts/docker/Dockerfile.copy +++ b/internal/scripts/docker/Dockerfile.copy @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base-unit-test -ARG MET_BASE_TAG=v1.0 +ARG MET_BASE_TAG=v1.1 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} MAINTAINER John Halley Gotway diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker index ec4bd202c3..c6674b231b 100644 --- a/internal/scripts/environment/development.docker +++ b/internal/scripts/environment/development.docker @@ -24,7 +24,7 @@ export MET_FREETYPELIB=/usr/lib64 export MET_JASPERLIB=/usr/lib64 export MET_PYTHON=/usr/bin/python3 -export MET_PYTHON_CC="-I/usr/include/python3.6m -I/usr/include/python3.6m" +export MET_PYTHON_CC="-I/usr/include/python3.6m" export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm" # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag diff --git a/internal/test_unit/xml/unit_ioda2nc.xml b/internal/test_unit/xml/unit_ioda2nc.xml index 91f2a7cbb4..55165568bd 100644 --- a/internal/test_unit/xml/unit_ioda2nc.xml +++ b/internal/test_unit/xml/unit_ioda2nc.xml @@ -83,4 +83,42 @@ + + &MET_BIN;/ioda2nc + + STATION_ID + MASK_GRID + MASK_POLY + MESSAGE_TYPE + + \ + &DATA_DIR_OBS;/ioda/jopa_satwind_20210701T1200Z_out_0000_reduced.nc4 \ + &OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc \ + -config &CONFIG_DIR;/IODA2NCConfig_mask \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc + + + + + &MET_BIN;/ioda2nc + + STATION_ID + MASK_GRID + MASK_POLY + MESSAGE_TYPE + + \ + &DATA_DIR_OBS;/ioda/2021081612_sonde_small.nc \ + &OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc \ + -config &CONFIG_DIR;/IODA2NCConfig_mask \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc + + + diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 37c0380b6a..1141e2c75e 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -650,6 +650,7 @@ static const char conf_key_trunc_factor[] = "gaussian_trunc_factor"; static const char conf_key_eclv_points[] = "eclv_points"; static const char conf_key_var_name_map[] = "var_name_map"; static const char conf_key_metadata_map[] = "metadata_map"; +static const char conf_key_obs_to_qc_map[] = "obs_to_qc_map"; static const char conf_key_missing_thresh[] = "missing_thresh"; static const char conf_key_control_id[] = "control_id"; static const char conf_key_ens_member_ids[] = "ens_member_ids"; @@ -897,6 +898,7 @@ static const char conf_key_do_polylines_flag [] = "do_polylines"; // PB2NC specific parameter key names // +static const char conf_key_datetime[] = "datetime"; static const char conf_key_station_id[] = "station_id"; static const char conf_key_elevation_range[] = "elevation_range"; static const char conf_key_pb_report_type[] = "pb_report_type"; diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc index e5747c8239..80dee7748f 100644 --- a/src/basic/vx_config/config_util.cc +++ b/src/basic/vx_config/config_util.cc @@ -1099,6 +1099,13 @@ map parse_conf_obs_name_map(Dictionary *dict) { /////////////////////////////////////////////////////////////////////////////// +map parse_conf_obs_to_qc_map(Dictionary *dict) { + const char *method_name = "parse_conf_obs_to_qc_map() -> "; + return parse_conf_key_values_map(dict, conf_key_obs_to_qc_map, method_name); +} + +/////////////////////////////////////////////////////////////////////////////// + map parse_conf_key_convert_map( Dictionary *dict, const char *conf_key_map_name, const char *caller) { Dictionary *map_dict = (Dictionary *) 0; diff --git a/src/basic/vx_config/config_util.h b/src/basic/vx_config/config_util.h index 02c712e715..9debcd2d49 100644 --- a/src/basic/vx_config/config_util.h +++ b/src/basic/vx_config/config_util.h @@ -52,9 +52,9 @@ extern NumArray parse_conf_eclv_points(Dictionary *dict); extern ClimoCDFInfo parse_conf_climo_cdf(Dictionary *dict); extern TimeSummaryInfo parse_conf_time_summary(Dictionary *dict); extern std::map parse_conf_key_value_map( - Dictionary *dict, const char *conf_key_map_name, const char *caller=0); + Dictionary *dict, const char *conf_key_map_name, const char *caller=0); extern void parse_add_conf_key_value_map( - Dictionary *dict, const char *conf_key_map_name, std::map *m); + Dictionary *dict, const char *conf_key_map_name, std::map *m); extern std::map parse_conf_message_type_map(Dictionary *dict); extern std::map @@ -62,8 +62,11 @@ extern std::map extern std::map parse_conf_metadata_map(Dictionary *dict); extern std::map parse_conf_obs_name_map(Dictionary *dict); -extern std::map parse_conf_key_convert_map( - Dictionary *dict, const char *conf_key_map_name, const char *caller=0); +extern std::map + parse_conf_obs_to_qc_map(Dictionary *dict); +extern std::map + parse_conf_key_convert_map( + Dictionary *dict, const char *conf_key_map_name, const char *caller=0); extern BootInfo parse_conf_boot(Dictionary *dict); extern RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out = default_dictionary_error_out); extern InterpInfo parse_conf_interp(Dictionary *dict, const char *); diff --git a/src/libcode/vx_data2d_factory/is_netcdf_file.cc b/src/libcode/vx_data2d_factory/is_netcdf_file.cc index 3e72029514..b14fa51fa6 100644 --- a/src/libcode/vx_data2d_factory/is_netcdf_file.cc +++ b/src/libcode/vx_data2d_factory/is_netcdf_file.cc @@ -40,9 +40,6 @@ static const char netcdf_magic [] = "CDF"; static const char hdf_magic [] = "HDF"; static const int netcdf_magic_len = m_strlen(netcdf_magic); -static const string nccf_att_name = "Conventions"; -static const string nccf_att_name_l = "conventions"; -static const string nccf_att_name_U = "CONVENTIONS"; static const string nccf_att_value = "CF-"; static const string nccf_att_value2 = "CF "; static const string nccf_att_value3 = "COARDS"; @@ -96,9 +93,7 @@ bool is_nccf_file(const char * filename) NcFile *nc_file = open_ncfile(filename); if (!IS_INVALID_NC_P(nc_file)) { - bool found = get_global_att(nc_file, nccf_att_name, att_val); - if (!found) found = get_global_att(nc_file, nccf_att_name_l, att_val); - if (!found) found = get_global_att(nc_file, nccf_att_name_U, att_val); + bool found = get_cf_conventions(nc_file, att_val); // "Conventions" attrribute if (found) { status = (att_val.compare(0, nccf_att_value.length(), nccf_att_value) == 0 || diff --git a/src/libcode/vx_nc_obs/nc_obs_util.cc b/src/libcode/vx_nc_obs/nc_obs_util.cc index 73db28167a..8f5a019280 100644 --- a/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -480,6 +480,9 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { obs_dim = get_nc_dim(f_in, nc_dim_nobs); // Observation array length hdr_dim = get_nc_dim(f_in, nc_dim_nhdr); // Header array length + use_var_id = false; + get_global_att(f_in, nc_att_use_var_id, use_var_id); + // Get netCDF header variables hdr_typ_var = get_var(f_in, nc_var_hdr_typ); // Message type (String or int) hdr_sid_var = get_var(f_in, nc_var_hdr_sid); // Station ID (String or int) @@ -503,10 +506,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { obs_arr_var = get_var(f_in, nc_var_obs_arr); } else { obs_hid_var = ncVar; // Obs. header id array - ncVar = get_var(f_in, nc_var_obs_gc); - if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array - ncVar = get_var(f_in, nc_var_obs_vid); - if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array + if (use_var_id) { + ncVar = get_var(f_in, nc_var_obs_vid); + if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array + } + else { + ncVar = get_var(f_in, nc_var_obs_gc); + if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array + } obs_lvl_var = get_var(f_in, nc_var_obs_lvl); // Obs. pressure level array obs_hgt_var = get_var(f_in, nc_var_obs_hgt); // Obs. highth array obs_val_var = get_var(f_in, nc_var_obs_val); // Obs. value array @@ -516,12 +523,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { ncVar = get_var(f_in, nc_var_obs_qty_tbl); if (!IS_INVALID_NC(ncVar)) obs_qty_tbl_var = ncVar; - ncVar = get_var(f_in, nc_var_obs_var); - if (!IS_INVALID_NC(ncVar)) obs_var = ncVar; - ncVar = get_var(f_in, nc_var_unit); - if (!IS_INVALID_NC(ncVar)) unit_var = ncVar; - ncVar = get_var(f_in, nc_var_desc); - if (!IS_INVALID_NC(ncVar)) desc_var = ncVar; + if (use_var_id) { + ncVar = get_var(f_in, nc_var_obs_var); + if (!IS_INVALID_NC(ncVar)) obs_var = ncVar; + ncVar = get_var(f_in, nc_var_unit); + if (!IS_INVALID_NC(ncVar)) unit_var = ncVar; + ncVar = get_var(f_in, nc_var_desc); + if (!IS_INVALID_NC(ncVar)) desc_var = ncVar; + } // PrepBufr only headers ncVar = get_var(f_in, nc_var_hdr_prpt_typ); @@ -531,12 +540,6 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { ncVar = get_var(f_in, nc_var_hdr_inst_typ); if (!IS_INVALID_NC(ncVar)) hdr_inst_typ_var = ncVar; - bool _use_var_id = false; - if (!get_global_att(f_in, nc_att_use_var_id, _use_var_id)) { - _use_var_id = IS_VALID_NC(obs_var); - } - - use_var_id = _use_var_id; } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 676fa75508..b312655396 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -165,9 +165,10 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value) { static const char *method_name = "get_att_value_chars(NcAtt) -> "; if (IS_VALID_NC_P(att)) { nc_type attType = GET_NC_TYPE_ID_P(att); - if (attType == NC_CHAR || attType == NC_STRING) { + if (attType == NC_CHAR) { try { - string att_value; + char att_value[tmp_buf_size]; + memset(att_value, 0, tmp_buf_size); att->getValues(att_value); value = att_value; } @@ -181,6 +182,33 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value) { << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; } } + else if (attType == NC_STRING) { + try { + string att_value; + att->getValues(att_value); + value = att_value; + } + catch (exceptions::NcChar ex) { + int num_elements_sub = 8096; + int num_elements = att->getAttLength();; + char *att_value[num_elements]; + for (int i = 0; i < num_elements; i++ ) { + att_value[i] = (char*) calloc(num_elements_sub, sizeof(char)); + } + try { + att->getValues(att_value); + value = att_value[0]; + } + catch (exceptions::NcException ex) { + mlog << Warning << "\n" << method_name + << "Exception: " << ex.what() << "\n" + << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" + << GET_NC_TYPE_NAME_P(att) << " type).\n" + << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; + } + for (int i = 0; i < num_elements; i++ ) delete att_value[i]; + } + } else { // MET-788: to handle a custom modified NetCDF mlog << Error << "\n" << method_name << "Please convert data type of \"" << GET_NC_NAME_P(att) @@ -336,6 +364,23 @@ bool get_att_no_leap_year(const NcVar *var) { return no_leap_year; } +//////////////////////////////////////////////////////////////////////// + +bool get_cf_conventions(const netCDF::NcFile *nc, ConcatString& conventions_value) { + bool has_attr = false; + multimap::iterator it_att; + multimap map_attrs = nc->getAtts(); + for (it_att = map_attrs.begin(); it_att != map_attrs.end(); it_att++) { + if (to_lower(it_att->first) == to_lower(cf_att_name)) { + NcGroupAtt cf_att = it_att->second; + if (IS_VALID_NC(cf_att)) + has_attr = get_att_value_chars(&cf_att, conventions_value); + } + } + return has_attr; +} + + //////////////////////////////////////////////////////////////////////// ConcatString get_log_msg_for_att(const NcVarAtt *att) { @@ -354,7 +399,7 @@ ConcatString get_log_msg_for_att(const NcVarAtt *att) { //////////////////////////////////////////////////////////////////////// ConcatString get_log_msg_for_att(const NcVarAtt *att, string var_name, - const ConcatString att_name) { + const ConcatString att_name) { ConcatString log_msg; log_msg << "can't read attribute" << " \"" << ((att_name.length() > 0) ? att_name.c_str() : GET_SAFE_NC_NAME_P(att)) @@ -445,7 +490,7 @@ NcGroupAtt *get_nc_att(const NcFile * nc, const ConcatString &att_name, bool exi //////////////////////////////////////////////////////////////////////// bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, - ConcatString &att_val, bool exit_on_error) { + ConcatString &att_val, int grp_id, bool exit_on_error) { bool status = false; NcVarAtt *att = (NcVarAtt *) 0; @@ -833,28 +878,49 @@ void add_att(NcVar *var, const string &att_name, const double att_val) { //////////////////////////////////////////////////////////////////////// -int get_var_names(NcFile *nc, StringArray *varNames) { +int get_var_names(NcFile *nc, StringArray *var_names) { - int i, varCount; NcVar var; + int i = 0; + int var_count = nc->getVarCount(); - varCount = nc->getVarCount(); - - i = 0; - multimap::iterator itVar; multimap mapVar = GET_NC_VARS_P(nc); - for (itVar = mapVar.begin(); itVar != mapVar.end(); ++itVar) { - var = (*itVar).second; - varNames->add(var.getName()); + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + var = (*it_var).second; + var_names->add(var.getName()); i++; } - if (i != varCount) { + if (i != var_count) { mlog << Error << "\n\tget_var_names() -> " - << "does not match array, allocated " << varCount << " but assigned " + << "does not match array, allocated " << var_count << " but assigned " << i << ".\n\n"; } - return(varCount); + return(var_count); +} + +//////////////////////////////////////////////////////////////////////// + +int get_var_names(NcFile *nc, StringArray *var_names, StringArray &group_names) { + + NcVar var; + NcGroup nc_group; + int var_count = 0; + multimap var_map; + multimap::iterator it_var; + + for (int idx=0; idxadd(it_var->first); + var_count++; + } + } + } + return(var_count); } //////////////////////////////////////////////////////////////////////// @@ -1657,6 +1723,14 @@ bool get_nc_data(NcVar *var, char *data) { //////////////////////////////////////////////////////////////////////// +bool get_nc_data(NcVar *var, char **data) { + bool return_status = get_nc_data_t(var, data); + + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + bool get_nc_data(NcVar *var, uchar *data) { bool return_status = false; int data_type = GET_NC_TYPE_ID_P(var); @@ -2114,6 +2188,18 @@ bool args_ok(const LongArray & a) { } //////////////////////////////////////////////////////////////////////// +// Continue even though not exists + +NcGroup get_nc_group(NcFile *nc, const char *group_name) { + NcGroup nc_group; + multimap group_map = nc->getGroups(); + multimap::iterator it = group_map.find(group_name); + if (it != group_map.end()) nc_group = it->second; + return nc_group; +} + +//////////////////////////////////////////////////////////////////////// +// Exit if exists but invalid NcVar get_var(NcFile *nc, const char *var_name) { string new_var_name = var_name; @@ -2123,23 +2209,57 @@ NcVar get_var(NcFile *nc, const char *var_name) { // Retrieve the variable from the NetCDF file. // NcVar var; - multimap varMap = GET_NC_VARS_P(nc); - multimap::iterator it = varMap.find(new_var_name); - if (it != varMap.end()) { - NcVar tmpVar = it->second; - if(IS_INVALID_NC(tmpVar)) { - mlog << Error << "\nget_var() -> " + multimap var_map = GET_NC_VARS_P(nc); + multimap::iterator it = var_map.find(new_var_name); + if (it != var_map.end()) { + var = it->second; + if(IS_INVALID_NC(var)) { + mlog << Error << "\nget_var(var_name) -> " << "can't read \"" << new_var_name << "\" variable.\n\n"; exit(1); } + } - var = tmpVar; + return(var); +} + +//////////////////////////////////////////////////////////////////////// +// Exit if exists but invalid + +NcVar get_var(NcFile *nc, const char *var_name, const char *group_name) { + string nc_var_name; + string new_var_name = var_name; + patch_nc_name(&new_var_name); + + // + // Retrieve the variable from the NetCDF file. + // + NcVar var; + multimap var_map; + NcGroup nc_group = get_nc_group(nc, group_name); + if (IS_VALID_NC(nc_group)) { + nc_var_name = new_var_name; + var_map = GET_NC_VARS(nc_group); + } + else { // This is for IODA data format 1.0 + nc_var_name = new_var_name + "@" + group_name; + var_map = GET_NC_VARS_P(nc); + } + multimap::iterator it = var_map.find(nc_var_name); + if (it != var_map.end()) { + var = it->second; + if(IS_INVALID_NC(var)) { + mlog << Error << "\nget_var(var_name, group_name) -> " + << "can't read \"" << new_var_name << "\" variable.\n\n"; + exit(1); + } } return(var); } //////////////////////////////////////////////////////////////////////// +// Continue even though not exists NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { string new_var_name = var_name; @@ -2151,7 +2271,7 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { NcVar var = nc->getVar(new_var_name); if(IS_INVALID_NC(var)) { ConcatString log_message; - log_message << "\nget_nc_var(NcFile) --> The variable \"" + log_message << "\nget_nc_var(var_name) --> The variable \"" << new_var_name << "\" does not exist!\n\n"; if (log_as_error) mlog << Error << log_message; @@ -2162,6 +2282,60 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { return(var); } +//////////////////////////////////////////////////////////////////////// +// Continue even though not exists + +NcVar get_nc_var(NcFile *nc, const ConcatString &var_name, bool log_as_error) { + return get_nc_var(nc, var_name.c_str(), log_as_error); +} + +//////////////////////////////////////////////////////////////////////// +// Continue even though not exists + +NcVar get_nc_var(NcFile *nc, const char *var_name, const char *group_name, + bool log_as_error) { + string nc_var_name; + string new_var_name = var_name; + patch_nc_name(&new_var_name); + + // + // Retrieve the variable from the NetCDF file. + // + NcVar var; + multimap var_map; + NcGroup nc_group = get_nc_group(nc, group_name); + if (IS_VALID_NC(nc_group)) { + nc_var_name = new_var_name; + var_map = GET_NC_VARS(nc_group); + } + else { // This is for IODA data format 1.0 + nc_var_name = new_var_name + "@" + group_name; + var_map = GET_NC_VARS_P(nc); + } + multimap::iterator it = var_map.find(nc_var_name); + if (it != var_map.end()) var = it->second; + + if(IS_INVALID_NC(var)) { + ConcatString log_message; + log_message << "\nget_nc_var(var_name, group_name) --> The variable \"" + << nc_var_name << "\" does not exist!\n\n"; + if (log_as_error) + mlog << Error << log_message; + else + mlog << Warning << log_message; + } + + return(var); +} + +//////////////////////////////////////////////////////////////////////// +// Continue even though not exists + +NcVar get_nc_var(NcFile *nc, const ConcatString &var_name, const char *group_name, + bool log_as_error) { + return get_nc_var(nc, var_name.c_str(), group_name, log_as_error); +} + //////////////////////////////////////////////////////////////////////// void copy_nc_att_byte(NcFile *nc_to, NcGroupAtt *from_att) { @@ -2506,8 +2680,8 @@ void copy_nc_atts(NcFile *nc_from, NcFile *nc_to, const bool all_attrs) { for (multimap::iterator itr = ncAttMap.begin(); itr != ncAttMap.end(); ++itr) { if (all_attrs || - ( (itr->first != "Conventions") - && (itr->first != "missing_value") ) ) { + ( (itr->first != cf_att_name) + && (itr->first != missing_value_att_name) ) ) { NcGroupAtt *from_att = &(itr->second); int dataType = GET_NC_TYPE_ID_P(from_att); switch (dataType) { @@ -2654,6 +2828,7 @@ void copy_nc_data_short(NcVar *var_from, NcVar *var_to, int data_size) { delete[] data; } +//////////////////////////////////////////////////////////////////////// void copy_nc_var_data(NcVar *var_from, NcVar *var_to) { const string method_name = "copy_nc_var_data()"; @@ -2698,13 +2873,60 @@ void copy_nc_var_dims(NcVar *var_from, NcVar *var_to) { //////////////////////////////////////////////////////////////////////// -bool has_var(NcFile *nc, const char * var_name) { - NcVar v = get_var(nc, var_name); +bool has_nc_group(NcFile *nc, const char *group_name) { + multimap group_map = nc->getGroups(); + multimap::iterator it = group_map.find(group_name); + return (it != group_map.end()); +} + +//////////////////////////////////////////////////////////////////////// + +bool has_var(NcFile *nc, const char *var_name) { + string new_var_name = var_name; + patch_nc_name(&new_var_name); + NcVar v = get_var(nc, new_var_name.c_str()); return IS_VALID_NC(v); } //////////////////////////////////////////////////////////////////////// +bool has_var(NcFile *nc, const ConcatString var_name) { + return has_var(nc, var_name.c_str()); +} + +//////////////////////////////////////////////////////////////////////// + +bool has_var(NcFile *nc, const char *var_name, const char *group_name) { + string nc_var_name; + string new_var_name = var_name; + patch_nc_name(&new_var_name); + + // + // Retrieve the variable from the NetCDF file. + // + multimap var_map; + NcGroup nc_group = get_nc_group(nc, group_name); + if (IS_VALID_NC(nc_group)) { + nc_var_name = new_var_name; + var_map = GET_NC_VARS(nc_group); + } + else { // This is for IODA data format 1.0 + nc_var_name = new_var_name + "@" + group_name; + var_map = GET_NC_VARS_P(nc); + } + multimap::iterator it = var_map.find(nc_var_name); + + return (it != var_map.end()); +} + +//////////////////////////////////////////////////////////////////////// + +bool has_var(NcFile *nc, const ConcatString var_name, const char *group_name) { + return has_var(nc, var_name.c_str(), group_name); +} + +//////////////////////////////////////////////////////////////////////// + NcVar add_var(NcFile *nc, const string &var_name, const NcType ncType, const int deflate_level) { vector ncDimVector; string new_var_name = var_name; @@ -3009,26 +3231,24 @@ vector get_dims(const NcVar *var, int *dim_count) { //////////////////////////////////////////////////////////////////////// bool is_nc_name_lat(const ConcatString name) { - bool is_latitude = (name == "lat" || name == "LAT" - || name == "Lat" || name == "Latitude" - || name == "latitude" || name == "LATITUDE"); + ConcatString name_l = to_lower(name); + bool is_latitude = (name_l == "lat" || name_l == "latitude"); return is_latitude; } //////////////////////////////////////////////////////////////////////// bool is_nc_name_lon(const ConcatString name) { - bool is_longitude = (name == "lon" || name == "LON" - || name == "Lon" || name == "Longitude" - || name == "longitude" || name == "LONGITUDE"); + ConcatString name_l = to_lower(name); + bool is_longitude = (name_l == "lon" || name_l == "longitude"); return is_longitude; } //////////////////////////////////////////////////////////////////////// bool is_nc_name_time(const ConcatString name) { - bool is_time = (name == "t" || name == "time" || name == "Time" || name == "TIME" - || name == "datetime" || name == "Datetime" || name == "DATETIME"); + ConcatString name_l = to_lower(name); + bool is_time = (name == "t" || name_l == "time" || name_l == "datetime"); return is_time; } @@ -3061,26 +3281,26 @@ NcVar get_nc_var_lat(const NcFile *nc) { multimap mapVar = GET_NC_VARS_P(nc); static const char *method_name = "get_nc_var_lat() "; - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - ConcatString name = (*itVar).first; + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + ConcatString name = (*it_var).first; //if (is_nc_name_lat(name)) found = true; - if (get_var_standard_name(&(*itVar).second, name)) { + if (get_var_standard_name(&(*it_var).second, name)) { if (is_nc_name_lat(name)) found = true; } - if (!found && get_var_units(&(*itVar).second, name)) { + if (!found && get_var_units(&(*it_var).second, name)) { if (is_nc_unit_latitude(name.c_str())) { - if (get_nc_att_value(&(*itVar).second, axis_att_name, name)) { + if (get_nc_att_value(&(*it_var).second, axis_att_name, name)) { if (is_nc_attr_lat(name)) found = true; } - else if (get_nc_att_value(&(*itVar).second, + else if (get_nc_att_value(&(*it_var).second, coordinate_axis_type_att_name, name)) { if (is_nc_attr_lat(name)) found = true; } } } if (found) { - var = (*itVar).second; + var = (*it_var).second; break; } } @@ -3103,26 +3323,26 @@ NcVar get_nc_var_lon(const NcFile *nc) { multimap mapVar = GET_NC_VARS_P(nc); static const char *method_name = "get_nc_var_lon() "; - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - ConcatString name = (*itVar).first; + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + ConcatString name = (*it_var).first; //if (is_nc_name_lon(name)) found = true; - if (get_var_standard_name(&(*itVar).second, name)) { + if (get_var_standard_name(&(*it_var).second, name)) { if (is_nc_name_lon(name)) found = true; } - if (!found && get_var_units(&(*itVar).second, name)) { + if (!found && get_var_units(&(*it_var).second, name)) { if (is_nc_unit_longitude(name.c_str())) { - if (get_nc_att_value(&(*itVar).second, axis_att_name, name)) { + if (get_nc_att_value(&(*it_var).second, axis_att_name, name)) { if (is_nc_attr_lon(name)) found = true; } - else if (get_nc_att_value(&(*itVar).second, + else if (get_nc_att_value(&(*it_var).second, coordinate_axis_type_att_name, name)) { if (is_nc_attr_lon(name)) found = true; } } } if (found) { - var = (*itVar).second; + var = (*it_var).second; break; } } @@ -3145,28 +3365,28 @@ NcVar get_nc_var_time(const NcFile *nc) { multimap mapVar = GET_NC_VARS_P(nc); static const char *method_name = "get_nc_var_time() "; - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - ConcatString name = (*itVar).first; + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + ConcatString name = (*it_var).first; //if (is_nc_name_time(name)) found = true; - if (get_var_standard_name(&(*itVar).second, name)) { + if (get_var_standard_name(&(*it_var).second, name)) { if (is_nc_name_time(name)) found = true; mlog << Debug(7) << method_name << "checked variable \"" << name << "\" is_time: " << found << "\n"; } - if (!found && get_var_units(&(*itVar).second, name)) { + if (!found && get_var_units(&(*it_var).second, name)) { if (is_nc_unit_time(name.c_str())) { - if (get_nc_att_value(&(*itVar).second, axis_att_name, name)) { + if (get_nc_att_value(&(*it_var).second, axis_att_name, name)) { if (is_nc_attr_time(name)) found = true; } - else if (get_nc_att_value(&(*itVar).second, + else if (get_nc_att_value(&(*it_var).second, coordinate_axis_type_att_name, name)) { if (is_nc_attr_time(name)) found = true; } } } if (found) { - var = (*itVar).second; + var = (*it_var).second; break; } } diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 199173e263..adb518d402 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -133,6 +133,7 @@ static const std::string axis_att_name = "axis"; static const std::string bounds_att_name = "bounds"; static const std::string coordinates_att_name = "coordinates"; static const std::string coordinate_axis_type_att_name = "_CoordinateAxisType"; +static const std::string cf_att_name = "Conventions"; static const std::string description_att_name = "description"; static const std::string fill_value_att_name = "_FillValue"; static const std::string grid_mapping_att_name = "grid_mapping"; @@ -179,14 +180,17 @@ extern long long get_att_value_llong (const netCDF::NcFile *, const ConcatString extern double get_att_value_double(const netCDF::NcFile *, const ConcatString& ); extern bool get_att_no_leap_year(const netCDF::NcVar *); -extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); -extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); +extern bool get_cf_conventions(const netCDF::NcFile *, ConcatString&); + +extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); +extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVarAtt *, std::string &); extern bool get_nc_att_value(const netCDF::NcVarAtt *, int &, bool exit_on_error = true); extern bool get_nc_att_value(const netCDF::NcVarAtt *, float &, bool exit_on_error = true); extern bool get_nc_att_value(const netCDF::NcVarAtt *, double &, bool exit_on_error = true); -extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, ConcatString &, bool exit_on_error = false); +extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, ConcatString &, + int nc_id=0, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, int &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, float &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, double &, bool exit_on_error = false); @@ -216,7 +220,8 @@ extern void add_att(netCDF::NcVar *, const std::string &, const int ); extern void add_att(netCDF::NcVar *, const std::string &, const float ); extern void add_att(netCDF::NcVar *, const std::string &, const double); -extern int get_var_names(netCDF::NcFile *, StringArray *varNames); +extern int get_var_names(netCDF::NcFile *, StringArray *var_names); +extern int get_var_names(netCDF::NcFile *, StringArray *var_names, StringArray &group_names); extern bool get_var_att_float (const netCDF::NcVar *, const ConcatString &, float &); extern bool get_var_att_double(const netCDF::NcVar *, const ConcatString &, double &); @@ -250,6 +255,7 @@ extern ConcatString* get_string_val(netCDF::NcVar *var, const int index, const i extern bool get_nc_data(netCDF::NcVar *, int *data); extern bool get_nc_data(netCDF::NcVar *, char *data); +extern bool get_nc_data(netCDF::NcVar *, char **data); extern bool get_nc_data(netCDF::NcVar *, uchar *data); extern bool get_nc_data(netCDF::NcVar *, float *data); extern bool get_nc_data(netCDF::NcVar *, double *data); @@ -315,17 +321,31 @@ extern bool put_nc_data_with_dims(netCDF::NcVar *, const double *data, const int extern bool put_nc_data_with_dims(netCDF::NcVar *, const double *data, const long len0, const long len1=0, const long len2=0); -extern netCDF::NcVar get_var(netCDF::NcFile *, const char * var_name); // exit if not exists -extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char * var_name, bool log_as_error=false); // continue even though not exists - -extern netCDF::NcVar *copy_nc_var(netCDF::NcFile *, netCDF::NcVar *, const int deflate_level=DEF_DEFLATE_LEVEL, const bool all_attrs=true); +extern netCDF::NcGroup get_nc_group(netCDF::NcFile *, const char *group_name); // continue even though not exists + +extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name); // exit if exists but invalid +extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name, + const char *group_name); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char *var_name, + bool log_as_error=false); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, + bool log_as_error=false); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char *var_name, + const char *group_name, bool log_as_error=false); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, + const char *group_name, bool log_as_error=false); // continue even though not exists + +extern netCDF::NcVar *copy_nc_var(netCDF::NcFile *, netCDF::NcVar *, + const int deflate_level=DEF_DEFLATE_LEVEL, const bool all_attrs=true); extern void copy_nc_att(netCDF::NcFile *, netCDF::NcVar *, const ConcatString attr_name); extern void copy_nc_att( netCDF::NcVar *, netCDF::NcVar *, const ConcatString attr_name); extern void copy_nc_atts(netCDF::NcFile *, netCDF::NcFile *, const bool all_attrs=true); extern void copy_nc_atts( netCDF::NcVar *, netCDF::NcVar *, const bool all_attrs=true); extern void copy_nc_var_data(netCDF::NcVar *, netCDF::NcVar *); -extern bool has_var(netCDF::NcFile *, const char * var_name); +extern bool has_nc_group(netCDF::NcFile *, const char *group_name); +extern bool has_var(netCDF::NcFile *, const char *var_name); +extern bool has_var(netCDF::NcFile *, const char *var_name, const char *group_name); extern netCDF::NcVar add_var(netCDF::NcFile *, const std::string &, const netCDF::NcType, const int deflate_level=DEF_DEFLATE_LEVEL); extern netCDF::NcVar add_var(netCDF::NcFile *, const std::string &, const netCDF::NcType, const netCDF::NcDim, const int deflate_level=DEF_DEFLATE_LEVEL); diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp index ceb5bd2807..f7c9c553a6 100644 --- a/src/libcode/vx_nc_util/nc_utils.hpp +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -46,12 +46,21 @@ extern void set_def_fill_value(unsigned short *val); template bool get_att_num_value_(const netCDF::NcAtt *att, T &att_val, int matching_type) { - bool status = false; - if (IS_VALID_NC_P(att)) { + bool status = IS_VALID_NC_P(att); + if (status) { int nc_type_id = GET_NC_TYPE_ID_P(att); if (matching_type == nc_type_id) { att->getValues(&att_val); - status = true; + } + else if (NC_FLOAT == nc_type_id) { + float att_value; + att->getValues(&att_value); + att_val = att_value; + } + else if (NC_DOUBLE == nc_type_id) { + double att_value; + att->getValues(&att_value); + att_val = att_value; } else if (NC_CHAR == nc_type_id) { std::string att_value; @@ -62,7 +71,6 @@ bool get_att_num_value_(const netCDF::NcAtt *att, T &att_val, int matching_type) att_val = (double)atof(att_value.c_str()); else // if (matching_type == NC_INT) att_val = atoi(att_value.c_str()); - status = true; } } return(status); @@ -74,8 +82,6 @@ template bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, T &att_val, bool exit_on_error, T bad_data, const char *caller_name) { - bool status = false; - // Initialize att_val = bad_data; @@ -83,7 +89,7 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, // Retrieve the NetCDF variable attribute. // netCDF::NcVarAtt *att = get_nc_att(var, att_name); - status = get_att_value((netCDF::NcAtt *)att, att_val); + bool status = get_att_value((netCDF::NcAtt *)att, att_val); if (!status) { mlog << Error << "\n" << caller_name << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); @@ -102,7 +108,6 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, template bool get_nc_att_value_(const netCDF::NcVarAtt *att, T &att_val, bool exit_on_error, T bad_data, const char *caller_name) { - bool status = true; // Initialize att_val = bad_data; @@ -110,7 +115,7 @@ bool get_nc_att_value_(const netCDF::NcVarAtt *att, T &att_val, bool exit_on_err // // Retrieve the NetCDF variable attribute. // - status = get_att_value((netCDF::NcAtt *)att, att_val); + bool status = get_att_value((netCDF::NcAtt *)att, att_val); if (!status) { mlog << Error << "\n" << caller_name << get_log_msg_for_att(att); @@ -259,12 +264,10 @@ void apply_scale_factor_(T *data, const int cell_count, template bool get_nc_data_t(netCDF::NcVar *var, T *data) { - bool return_status = false; + bool return_status = IS_VALID_NC_P(var); - if (IS_VALID_NC_P(var)) { + if (return_status) { var->getVar(data); - - return_status = true; } return(return_status); } diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index 1aaef67ba1..ad0c90353f 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -63,6 +63,15 @@ static const char * DEF_CONFIG_NAME = "MET_BASE/config/IODA2NCConfig_default"; static const char *program_name = "ioda2nc"; static const int REJECT_DEBUG_LEVEL = 9; +static const int string_data_len = 512; + +static const char *metadata_group_name = "MetaData"; +static const char *qc_group_name = "QCFlags"; +static const char *qc_postfix = "PreQC"; +static const char *obs_group_name = "ObsValue"; +static const char *derived_obs_group_name = "DerivedObsValue"; + +enum e_ioda_format { ioda_v1, ioda_v2 }; //////////////////////////////////////////////////////////////////////// @@ -74,7 +83,8 @@ static const int REJECT_DEBUG_LEVEL = 9; static StringArray ioda_files; static StringArray core_dims; -static StringArray core_vars; +static StringArray core_dims_v1; +static StringArray core_meta_vars; // Output NetCDF file name static ConcatString ncfile; @@ -154,13 +164,15 @@ static void set_valid_beg_time(const StringArray &); static void set_valid_end_time(const StringArray &); static void set_verbosity(const StringArray &); -static bool check_core_data(const bool, const bool, StringArray &, StringArray &); +static bool check_core_data(const bool, const bool, StringArray &, StringArray &, e_ioda_format); static bool check_missing_thresh(float value); static ConcatString find_meta_name(StringArray, StringArray); -static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, const int); -static bool get_meta_data_strings(NcFile *, const ConcatString, char *); -static bool get_obs_data_float(NcFile *, const ConcatString, - NcVar *, float *, int *, const int); +static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, + const int); +static bool get_meta_data_strings(NcVar &, char *); +static bool get_meta_data_strings(NcVar &, char **); +static bool get_obs_data_float(NcFile *, const ConcatString, NcVar *, + float *, int *, const int, const e_ioda_format); static bool has_postfix(std::string const &, std::string const &); //////////////////////////////////////////////////////////////////////// @@ -214,15 +226,18 @@ void initialize() { nc_point_obs.init_buffer(); core_dims.clear(); - core_dims.add("nvars"); core_dims.add("nlocs"); - //core_dims.add("nstring"); - //core_dims.add("ndatetime"); + + core_dims_v1.clear(); + core_dims_v1.add("nvars"); + core_dims_v1.add("nlocs"); + core_dims_v1.add("nstring"); + //core_dims_v1.add("ndatetime"); - core_vars.clear(); - core_vars.add("datetime"); - core_vars.add("latitude"); - core_vars.add("longitude"); + core_meta_vars.clear(); + core_meta_vars.add("datetime"); + core_meta_vars.add("latitude"); + core_meta_vars.add("longitude"); summary_obs = new SummaryObs(); return; @@ -349,6 +364,9 @@ void process_ioda_file(int i_pb) { int rej_elv, rej_nobs; double x, y; + bool status; + bool is_time_offset = false; + bool is_time_string = false; unixtime file_ut; unixtime adjusted_file_ut; unixtime msg_ut, beg_ut, end_ut; @@ -411,39 +429,76 @@ void process_ioda_file(int i_pb) { file_name << ioda_files[i_pb]; int nrecs = 0; - StringArray var_names, dim_names; + int nstring, nvars; + StringArray dim_names; StringArray metadata_vars; StringArray obs_value_vars; - get_var_names(f_in, &var_names); + bool error_out = true; + e_ioda_format ioda_format = ioda_v2; + int nlocs = get_dim_value(f_in, "nlocs", error_out); // number of locations + + nvars = bad_data_int ; + nstring = string_data_len; + if (! has_nc_group(f_in, obs_group_name)) ioda_format = ioda_v1; + get_dim_names(f_in, &dim_names); - for(idx=0; idx= 8) { + for(idx=0; idx= 6) { - for(idx=0; idx 0 && nmsg < npbmsg) { npbmsg = (nmsg_percent > 0 && nmsg_percent <= 100) - ? (npbmsg * nmsg_percent / 100) : nmsg; + ? (npbmsg * nmsg_percent / 100) : nmsg; } - long lengths[2] = { nlocs, ndatetime }; - long offsets[2] = { 0, 0 }; float *hdr_lat_arr = new float[nlocs]; float *hdr_lon_arr = new float[nlocs]; float *hdr_elv_arr = new float[nlocs]; float *obs_pres_arr = new float[nlocs]; float *obs_hght_arr = new float[nlocs]; + float *hdr_time_arr = new float[nlocs]; char *hdr_vld_block = new char[nlocs*ndatetime]; - char *hdr_msg_types = 0; - char *hdr_station_ids = 0; + char *hdr_msg_types = NULL; + char *hdr_station_ids = NULL; + char **hdr_vld_block2 = NULL; + char **hdr_msg_types2 = NULL; + char **hdr_station_ids2 = NULL; vector v_qc_data; vector v_obs_data; + + if (is_time_string) { + hdr_vld_block2 = (char**) calloc(nlocs, sizeof(char*)); + for (int i=0; i= debug_level_for_performance) { @@ -605,6 +711,7 @@ void process_ioda_file(int i_pb) { for(idx=0; idx 0) { @@ -620,11 +727,25 @@ void process_ioda_file(int i_pb) { } } - char valid_time[ndatetime+1]; - m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), - ndatetime, method_name_s, "valid_time", true); - valid_time[ndatetime] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); + if (is_time_offset) { + msg_ut = add_to_unixtime(base_ut, sec_per_unit, + hdr_time_arr[i_read], no_leap_year); + } + else if (is_time_string) { + char valid_time[nstring+1]; + + m_strncpy(valid_time, (const char *)hdr_vld_block2[i_read], + nstring, method_name_s, "valid_time", true); + valid_time[nstring] = 0; + msg_ut = yyyymmddThhmmss_to_unix(valid_time); + } + else { + char valid_time[ndatetime+1]; + m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), + ndatetime, method_name_s, "valid_time", true); + valid_time[ndatetime] = 0; + msg_ut = yyyymmddThhmmss_to_unix(valid_time); + } // Check to make sure that the message time hasn't changed // from one IODA message to the next @@ -638,8 +759,7 @@ void process_ioda_file(int i_pb) { // Check if valid_beg_ut and valid_end_ut were set on the // command line. If so, use them. If not, use beg_ds and // end_ds. - if(valid_beg_ut != (unixtime) 0 || - valid_end_ut != (unixtime) 0) { + if(valid_beg_ut != (unixtime) 0 || valid_end_ut != (unixtime) 0) { beg_ut = valid_beg_ut; end_ut = valid_end_ut; } @@ -671,7 +791,13 @@ void process_ioda_file(int i_pb) { } if(has_msg_type) { - m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name_s, "hdr_typ"); + if (NULL != hdr_msg_types2) { + m_strncpy(hdr_typ, hdr_msg_types2[i_read], nstring, method_name_s, "hdr_typ2"); + } + else { + m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name_s, "hdr_typ"); + + } m_rstrip(hdr_typ, nstring); // If the message type is not listed in the configuration @@ -697,7 +823,12 @@ void process_ioda_file(int i_pb) { if(has_station_id) { char tmp_sid[nstring+1]; - m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name_s, "tmp_sid"); + if (NULL != hdr_station_ids2) { + m_strncpy(tmp_sid, hdr_station_ids2[i_read], nstring, method_name_s, "tmp_sid2"); + } + else { + m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name_s, "tmp_sid"); + } m_rstrip(tmp_sid, nstring, false); m_replace_char(tmp_sid, ' ', '_'); hdr_sid = tmp_sid; @@ -881,6 +1012,19 @@ void process_ioda_file(int i_pb) { delete [] obs_hght_arr; if (hdr_msg_types) delete [] hdr_msg_types; if (hdr_station_ids) delete [] hdr_station_ids; + if (NULL != hdr_msg_types2) { + for (int i=0; i " - << "core dimension \"" << core_dims[idx] << "\" is missing.\n\n"; + << "core dimension \"" << t_core_dims[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } - if(has_msg_type || has_station_id) { - if(!dim_names.has("nstring")) { - mlog << Error << "\n" << method_name << "-> " - << "core dimension \"nstring\" is missing.\n\n"; - is_netcdf_ready = false; + if (ioda_format == ioda_v1) { + if(has_msg_type || has_station_id) { + if(!dim_names.has("nstring")) { + mlog << Error << "\n" << method_name << "-> " + << "core dimension \"nstring\" is missing.\n\n"; + is_netcdf_ready = false; + } } } - for(int idx=0; idx 0) { + for (int idx2=0; idx2 < alt_names.n(); idx2++) { + if (core_meta_vars[idx] != alt_names[idx2]) { + found = metadata_vars.has(alt_names[idx2]); + if (found) break; + } + } + } + } + if(!found) { mlog << Error << "\n" << method_name << "-> " - << "core variable \"" << core_vars[idx] << "\" is missing.\n\n"; + << "core variable \"" << core_meta_vars[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } @@ -1123,10 +1284,9 @@ bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, ConcatString metadata_name = find_meta_name( metadata_vars, conf_info.metadata_map[metadata_key]); + if(metadata_name.length() > 0) { - ConcatString ioda_name = metadata_name; - ioda_name.add("@MetaData"); - NcVar meta_var = get_var(f_in, ioda_name.c_str()); + NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); if(IS_VALID_NC(meta_var)) { status = get_nc_data(&meta_var, metadata_buf, nlocs); if(!status) mlog << Debug(3) << method_name @@ -1148,19 +1308,15 @@ bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, //////////////////////////////////////////////////////////////////////// -bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, - char *metadata_buf) { +bool get_meta_data_strings(NcVar &var, char *metadata_buf) { bool status = false; static const char *method_name = "get_meta_data_strings() -> "; - ConcatString ioda_name = metadata_name; - ioda_name.add("@MetaData"); - NcVar meta_var = get_var(f_in, ioda_name.c_str()); - if(IS_VALID_NC(meta_var)) { - status = get_nc_data(&meta_var, metadata_buf); + if(IS_VALID_NC(var)) { + status = get_nc_data(&var, metadata_buf); if(!status) { mlog << Error << "\n" << method_name << " -> " - << "trouble getting " << metadata_name << "\n\n"; + << "trouble getting " << GET_NC_NAME(var) << "\n\n"; exit(1); } } @@ -1171,7 +1327,7 @@ bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, NcVar *obs_var, float *obs_buf, int *qc_buf, - const int nlocs) { + const int nlocs, const e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_obs_data_float() -> "; @@ -1185,21 +1341,37 @@ bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, << "trouble getting " << var_name << "\n\n"; } else mlog << Error << "\n" << method_name - << var_name << "@ObsValue does not exist!\n\n"; + << var_name << " does not exist!\n\n"; if(!status) exit(1); status = false; if(var_name.length() > 0) { - ConcatString ioda_name = var_name; - ioda_name.add("@PreQC"); - NcVar qc_var = get_var(f_in, ioda_name.c_str()); + ConcatString qc_name = var_name; + ConcatString qc_group = qc_postfix; + if (ioda_format == ioda_v2) { + NcGroup nc_grp = get_nc_group(f_in, qc_postfix); + if (IS_INVALID_NC(nc_grp)) qc_group = qc_group_name; + StringArray qc_names = conf_info.obs_to_qc_map[var_name]; + if (0 < qc_names.n()) { + for (int idx=0; idx obs_name_map; map message_type_map; map metadata_map; + map obs_to_qc_map; StringArray surface_message_types; TimeSummaryInfo timeSummaryInfo;