From b6bb5e7839cb8ea8fe58d312393b71f09c6fa823 Mon Sep 17 00:00:00 2001 From: hsoh-u Date: Fri, 28 May 2021 16:50:42 -0600 Subject: [PATCH] Feature 1581 api point obs (#1812) * #1581 Initial release * #1581 Added met_nc_point_obs.cc met_nc_point_obs.h * Removed nc_ in function names and moved them to the struct members * #1581 Added HDR_TYPE_ARR_LEN * #1581 Changed API calls (API names) * #1581 Cleanup * #1581 Removed duplicated definitions:hdr_arr_len, hdr_typ_arr_len, and obs_arr_len * #1581 Removed duplicated definitions:hdr_arr_len and obs_arr_len * #1581 Removed duplicated definitions: str_len, hdr_arr_len, and obs_arr_len * Added vx_nc_obs library * #1581 Using common APIs * #1581 Corrected API calls because of renaming for common APIs * #1581 Moved function from nc_obs_util to nc_point_obs2 * #1581renamed met_nc_point_obs to nc_point_obs * #1581 API ica changed from obs_vars to nc_point_obs * #1581 Initial release * #1581 Renamed from met_nc_point_obs to nc_point_obs * 1581 Renamed met_nc_point_obs to nc_point_obs * Per #1581, update the Makefile.am for lidar2nc to fix a linker error. Unfortunatley, the vx_config library now depends on the vx_gsl_prob library. threshold.cc in vx_config includes a call to normal_cdf_inv(double, double, double) which is defined in vx_gsl_prob. This adds to the complexity of dependencies for MET's libraries. Just linking to -lvx_gsl_prob one more time does fix the linker problem but doesn't solve the messy dependencies. * #1581 Added method for NcDataBuffer * Cleanup * #1581 Cleanup * #1581 Cleanup * #1591 Cleanup * #1591 Corrected API * #1581 Avoid reading PB header twice * #1581 Warning if PB header is not defined but read_pb_hdr_data is called * #1581 Cleanup libraries * 1581 cleanup * 1581 cleanup * 1581 cleanup * #1581 Cleanup for Fortify (removed unused variables) * #1581 Cleanup * #1581 Cleanup * #1581 Use MetNcPointObsIn instead of MetNcPointObs * #1581 Use MetNcPointObsOut instead of MetNcPointObs2Write * #1581 Separated nc_point_obs2.cc to nc_point_obs_in.cc and nc_point_obs_out.cc * #1581 Renamed nc_point_obs2.cc to nc_point_obs_in.cc And added add nc_point_obs_in.h nc_point_obs_out.h nc_point_obs_out.cc * #1581 Removed APIs related with writing point obs * #1581 Changed copyright years * #1581 Cleanup * #1581 Updated copyright year * #1581 Cleanup * #1581 Reanmed read_obs_data_strings to read_obs_data_table_lookups * #1581 Reanmed read_obs_data_strings to read_obs_data_table_lookups * #1581 Added more APIs Co-authored-by: Howard Soh Co-authored-by: John Halley Gotway --- met/src/libcode/vx_nc_obs/Makefile.am | 3 + met/src/libcode/vx_nc_obs/nc_obs_util.cc | 1913 ++++++++--------- met/src/libcode/vx_nc_obs/nc_obs_util.h | 277 +-- met/src/libcode/vx_nc_obs/nc_point_obs.cc | 167 ++ met/src/libcode/vx_nc_obs/nc_point_obs.h | 109 + met/src/libcode/vx_nc_obs/nc_point_obs_in.cc | 113 + met/src/libcode/vx_nc_obs/nc_point_obs_in.h | 58 + met/src/libcode/vx_nc_obs/nc_point_obs_out.cc | 607 ++++++ met/src/libcode/vx_nc_obs/nc_point_obs_out.h | 104 + met/src/libcode/vx_nc_obs/nc_summary.cc | 111 +- met/src/libcode/vx_nc_obs/nc_summary.h | 6 +- met/src/libcode/vx_nc_util/nc_utils.h | 1 + met/src/libcode/vx_nc_util/write_netcdf.cc | 41 - met/src/tools/core/ensemble_stat/Makefile.am | 8 +- .../tools/core/ensemble_stat/ensemble_stat.cc | 165 +- .../tools/core/ensemble_stat/ensemble_stat.h | 9 - met/src/tools/core/point_stat/Makefile.am | 10 +- met/src/tools/core/point_stat/point_stat.cc | 188 +- met/src/tools/core/point_stat/point_stat.h | 6 - met/src/tools/other/ascii2nc/Makefile.am | 12 +- met/src/tools/other/ascii2nc/file_handler.cc | 76 +- met/src/tools/other/ascii2nc/file_handler.h | 4 +- met/src/tools/other/ioda2nc/Makefile.am | 12 +- met/src/tools/other/ioda2nc/ioda2nc.cc | 57 +- met/src/tools/other/lidar2nc/Makefile.am | 3 + met/src/tools/other/lidar2nc/lidar2nc.cc | 110 +- met/src/tools/other/madis2nc/Makefile.am | 12 +- met/src/tools/other/madis2nc/madis2nc.cc | 79 +- met/src/tools/other/madis2nc/madis2nc.h | 4 - met/src/tools/other/pb2nc/Makefile.am | 11 +- met/src/tools/other/pb2nc/pb2nc.cc | 181 +- .../tools/other/plot_point_obs/Makefile.am | 16 +- .../other/plot_point_obs/plot_point_obs.cc | 263 +-- met/src/tools/other/point2grid/Makefile.am | 11 +- met/src/tools/other/point2grid/point2grid.cc | 127 +- 35 files changed, 2649 insertions(+), 2225 deletions(-) create mode 100644 met/src/libcode/vx_nc_obs/nc_point_obs.cc create mode 100644 met/src/libcode/vx_nc_obs/nc_point_obs.h create mode 100644 met/src/libcode/vx_nc_obs/nc_point_obs_in.cc create mode 100644 met/src/libcode/vx_nc_obs/nc_point_obs_in.h create mode 100644 met/src/libcode/vx_nc_obs/nc_point_obs_out.cc create mode 100644 met/src/libcode/vx_nc_obs/nc_point_obs_out.h diff --git a/met/src/libcode/vx_nc_obs/Makefile.am b/met/src/libcode/vx_nc_obs/Makefile.am index 67734a45f9..ba2fb8c910 100644 --- a/met/src/libcode/vx_nc_obs/Makefile.am +++ b/met/src/libcode/vx_nc_obs/Makefile.am @@ -13,5 +13,8 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_nc_obs.a libvx_nc_obs_a_SOURCES = \ nc_obs_util.cc nc_obs_util.h \ + nc_point_obs.cc nc_point_obs.h \ + nc_point_obs_in.cc nc_point_obs_in.h \ + nc_point_obs_out.cc nc_point_obs_out.h \ nc_summary.cc nc_summary.h libvx_nc_obs_a_CPPFLAGS = ${MET_CPPFLAGS} diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.cc b/met/src/libcode/vx_nc_obs/nc_obs_util.cc index 5cd8709097..805461b078 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -16,18 +16,14 @@ using namespace std; -//#include #include -//#include "vx_math.h" #include "vx_nc_util.h" #include "nc_obs_util.h" /////////////////////////////////////////////////////////////////////////////// -struct NcDataBuffer nc_data_buffer; -struct NcHeaderData hdr_data; float hdr_arr_block[NC_BUFFER_SIZE_32K][HDR_ARRAY_LEN]; /////////////////////////////////////////////////////////////////////////////// @@ -45,406 +41,614 @@ static const string err_msg_hdr_arr = "error writing the header array to the netCDF file\n\n"; /////////////////////////////////////////////////////////////////////////////// +// struct NcDataBuffer -bool add_nc_header_to_array (const char *hdr_typ, const char *hdr_sid, - const time_t hdr_vld, const float hdr_lat, - const float hdr_lon, const float hdr_elv) +NcDataBuffer::NcDataBuffer() { + reset_counters(); +} + +/////////////////////////////////////////////////////////////////////////////// + +void NcDataBuffer::reset_counters() { + processed_count = 0; + obs_count = 0; + obs_buf_size = 0; + cur_obs_idx = 0; + obs_data_idx = 0; + obs_data_offset = 0; + hdr_count = 0; + hdr_buf_size = 0; + cur_hdr_idx = 0; + hdr_data_idx = 0; + hdr_data_offset = 0; + pb_hdr_count = 0; + pb_hdr_data_offset = 0; +} + +/////////////////////////////////////////////////////////////////////////////// +// struct NcPointObsData + +NcPointObsData::NcPointObsData(): + obs_cnt(0), + obs_ids((int *)0), + obs_hids((int *)0), + obs_qids((int *)0), + obs_lvls((float *)0), + obs_hgts((float *)0), + obs_vals((float *)0), + obs_arr((float *)0), + is_obs_array(false) { - bool added = false; - bool new_vld = false; - - // Can't filter duplicated one because header index was - // assigned before checking - int hdr_index; - if (!hdr_data.typ_array.has(hdr_typ, hdr_index, false)) { - hdr_index = hdr_data.typ_array.n_elements(); - hdr_data.typ_array.add(hdr_typ); // Message type +} + +/////////////////////////////////////////////////////////////////////////////// + +void NcPointObsData::clear() { + obs_cnt = 0; + is_obs_array = false; + + clear_numbers(); + clear_strings(); +} + +/////////////////////////////////////////////////////////////////////////////// + +void NcPointObsData::clear_numbers() { + if (0 != obs_ids) { + delete [] obs_ids; + obs_ids = (int *)0; } - hdr_data.typ_idx_array.add(hdr_index); // Index of Message type - - if (!hdr_data.sid_array.has(hdr_sid, hdr_index, false)) { - hdr_index = hdr_data.sid_array.n_elements(); - hdr_data.sid_array.add(hdr_sid); // Station ID + if (0 != obs_hids) { + delete [] obs_hids; + obs_hids = (int *)0; } - hdr_data.sid_idx_array.add(hdr_index); // Index of Station ID - - if (hdr_data.min_vld_time == -1 || hdr_data.min_vld_time > hdr_vld) { - if (hdr_data.min_vld_time == -1) hdr_data.max_vld_time = hdr_vld; - hdr_data.min_vld_time = hdr_vld; - new_vld = true; + if (0 != obs_qids) { + delete [] obs_qids; + obs_qids = (int *)0; } - else if (hdr_data.max_vld_time < hdr_vld) { - hdr_data.max_vld_time = hdr_vld; - new_vld = true; + if (0 != obs_lvls) { + delete [] obs_lvls; + obs_lvls = (float *)0; } - if (new_vld || !hdr_data.vld_num_array.has(hdr_vld, hdr_index, false)) { - hdr_index = hdr_data.vld_array.n_elements(); - hdr_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); // Valid time - hdr_data.vld_num_array.add(hdr_vld); // Valid time + if (0 != obs_hgts) { + delete [] obs_hgts; + obs_hgts = (float *)0; + } + if (0 != obs_vals) { + delete [] obs_vals; + obs_vals = (float *)0; + } + if (0 != obs_arr) { + delete [] obs_arr; + obs_arr = (float *)0; } - hdr_data.vld_idx_array.add(hdr_index); // Index of Valid time - - hdr_data.lat_array.add(hdr_lat); // Latitude - hdr_data.lon_array.add(hdr_lon); // Longitude - hdr_data.elv_array.add(hdr_elv); // Elevation - nc_data_buffer.cur_hdr_idx++; - added = true; - return added; } /////////////////////////////////////////////////////////////////////////////// -bool add_nc_header_prepbufr (const int pb_report_type, - const int in_report_type, const int instrument_type) -{ - bool added = true; - // Can't filter duplicated one because header index was - // assigned before checking - hdr_data.prpt_typ_array.add(pb_report_type); - hdr_data.irpt_typ_array.add(in_report_type); - hdr_data.inst_typ_array.add(instrument_type); - return added; +void NcPointObsData::clear_strings() { + var_names.clear(); + qty_names.clear(); } -//////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +float NcPointObsData::get_obs_val(int index) { + float obs_val = (is_obs_array ? obs_arr[index*obs_cnt+4] : obs_vals[index]); + return obs_val; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { + bool succeed = true; + const char* method_name = "NcPointObsData::read_obs_data_numbers()"; -int check_nc_dims_vars(const NetcdfObsVars obs_vars) { - int exit_code = exit_code_no_error; - if(IS_INVALID_NC(obs_vars.strl_dim) || - IS_INVALID_NC(obs_vars.obs_dim) || - IS_INVALID_NC(obs_vars.hdr_dim)) { - exit_code = exit_code_no_dim; + clear_numbers(); + obs_cnt = obs_vars.obs_cnt; + + StringArray missing_vars; + StringArray failed_vars; + if (!IS_INVALID_NC(obs_vars.obs_arr_var)) { + is_obs_array = true; + obs_arr = new float[obs_cnt*OBS_ARRAY_LEN]; + if (!get_nc_data(&obs_vars.obs_arr_var, obs_arr)) { + succeed = false; + failed_vars.add(nc_var_obs_arr); + } } - else if(IS_INVALID_NC(obs_vars.hdr_typ_var) || - IS_INVALID_NC(obs_vars.hdr_sid_var) || - IS_INVALID_NC(obs_vars.hdr_vld_var)) { - exit_code = exit_code_no_hdr_vars; + else { + if (IS_INVALID_NC(obs_vars.obs_hid_var)) { + succeed = false; + missing_vars.add(nc_var_obs_hid); + } + else { + obs_hids = new int[obs_cnt]; + if (!get_nc_data(&obs_vars.obs_hid_var, obs_hids)) { + succeed = false; + failed_vars.add(nc_var_obs_hid); + } + } + if (IS_INVALID_NC(obs_vars.obs_lvl_var)) { + succeed = false; + missing_vars.add(nc_var_obs_lvl); + } + else { + obs_lvls = new float[obs_cnt]; + if (!get_nc_data(&obs_vars.obs_lvl_var, obs_lvls)) { + succeed = false; + failed_vars.add(nc_var_obs_lvl); + } + } + if (IS_INVALID_NC(obs_vars.obs_hgt_var)) { + succeed = false; + missing_vars.add(nc_var_obs_hgt); + } + else { + obs_hgts = new float[obs_cnt]; + if (!get_nc_data(&obs_vars.obs_hgt_var, obs_hgts)) { + succeed = false; + failed_vars.add(nc_var_obs_hgt); + } + } + if (IS_INVALID_NC(obs_vars.obs_val_var)) { + succeed = false; + missing_vars.add(nc_var_obs_val); + } + else { + obs_vals = new float[obs_cnt]; + if (!get_nc_data(&obs_vars.obs_val_var, obs_vals)) { + succeed = false; + failed_vars.add(nc_var_obs_val); + } + } + if (IS_VALID_NC(obs_vars.obs_gc_var)) { + obs_ids = new int[obs_cnt]; + if (!get_nc_data(&obs_vars.obs_gc_var, obs_ids)) { + succeed = false; + failed_vars.add(nc_var_obs_gc); + } + } + else if (IS_VALID_NC(obs_vars.obs_vid_var)) { + obs_ids = new int[obs_cnt]; + if (!get_nc_data(&obs_vars.obs_vid_var, obs_ids)) { + succeed = false; + failed_vars.add(nc_var_obs_vid); + } + } + else succeed = false; + } - else if((IS_INVALID_NC(obs_vars.obs_arr_var) && IS_INVALID_NC(obs_vars.obs_val_var))) { - exit_code = exit_code_no_obs_vars; + + for (int idx=0; idxmin_vld_time = -1; - my_hdr_data->max_vld_time = -1; - - my_hdr_data->typ_array.clear(); - my_hdr_data->sid_array.clear(); - my_hdr_data->vld_array.clear(); - my_hdr_data->vld_num_array.clear(); - my_hdr_data->typ_idx_array.clear(); - my_hdr_data->sid_idx_array.clear(); - my_hdr_data->vld_idx_array.clear(); - my_hdr_data->lat_array.clear(); - my_hdr_data->lon_array.clear(); - my_hdr_data->elv_array.clear(); - my_hdr_data->prpt_typ_array.clear(); - my_hdr_data->irpt_typ_array.clear(); - my_hdr_data->inst_typ_array.clear(); + if (stop && !succeed) exit(1); + return succeed; } + /////////////////////////////////////////////////////////////////////////////// +// struc NetcdfObsVars -long count_nc_headers(vector< Observation > &observations) +NetcdfObsVars::NetcdfObsVars() { - long nhdr = 0; - - string prev_header_type = ""; - string prev_station_id = ""; - time_t prev_valid_time = 0; - double prev_latitude = bad_data_double; - double prev_longitude = bad_data_double; - double prev_elevation = bad_data_double; - const string method_name = " count_nc_headers()"; - - for (vector< Observation >::iterator obs = observations.begin(); - obs != observations.end(); ++obs) - { - if (obs->getHeaderType() != prev_header_type || - obs->getStationId() != prev_station_id || - obs->getValidTime() != prev_valid_time || - !is_eq(obs->getLatitude(), prev_latitude) || - !is_eq(obs->getLongitude(), prev_longitude) || - !is_eq(obs->getElevation(), prev_elevation)) - { - nhdr++; - - prev_header_type = obs->getHeaderType(); - prev_station_id = obs->getStationId(); - prev_valid_time = obs->getValidTime(); - prev_latitude = obs->getLatitude(); - prev_longitude = obs->getLongitude(); - prev_elevation = obs->getElevation(); - } - //else mlog << Debug(7) << method_name - // << " FFF obs->getHeaderIndex(): " << obs->getHeaderIndex() - // << ", nhdr: " << nhdr << " count: " << count - // << "\n"; - obs->setHeaderIndex(nhdr-1); - } /* endfor - obs */ - - return nhdr; + reset(); } /////////////////////////////////////////////////////////////////////////////// -void create_nc_dimensions(NetcdfObsVars &obs_vars, NcFile *f_out) { - const string method_name = " create_nc_dimensions()"; +void NetcdfObsVars::create_dimensions(NcFile *f_out) { + const string method_name = " create_dimensions()"; mlog << Debug(7) << method_name << " is called" << "\n"; // Define netCDF dimensions - if (IS_INVALID_NC(obs_vars.strl_dim)) obs_vars.strl_dim = add_dim(f_out, nc_dim_mxstr, HEADER_STR_LEN); - if (IS_INVALID_NC(obs_vars.strl2_dim)) obs_vars.strl2_dim = add_dim(f_out, nc_dim_mxstr2, HEADER_STR_LEN2); - if (IS_INVALID_NC(obs_vars.strl3_dim)) obs_vars.strl3_dim = add_dim(f_out, nc_dim_mxstr3, HEADER_STR_LEN3); - if (IS_INVALID_NC(obs_vars.hdr_dim) && obs_vars.hdr_cnt > 0) { - obs_vars.hdr_dim = add_dim(f_out, nc_dim_nhdr, obs_vars.hdr_cnt); + if (IS_INVALID_NC(strl_dim)) strl_dim = add_dim(f_out, nc_dim_mxstr, HEADER_STR_LEN); + if (IS_INVALID_NC(strl2_dim)) strl2_dim = add_dim(f_out, nc_dim_mxstr2, HEADER_STR_LEN2); + if (IS_INVALID_NC(strl3_dim)) strl3_dim = add_dim(f_out, nc_dim_mxstr3, HEADER_STR_LEN3); + if (IS_INVALID_NC(hdr_dim) && hdr_cnt > 0) { + hdr_dim = add_dim(f_out, nc_dim_nhdr, hdr_cnt); } - if (IS_INVALID_NC(obs_vars.obs_dim)) { - if (obs_vars.obs_cnt > 0) obs_vars.obs_dim = add_dim(f_out, nc_dim_nobs, obs_vars.obs_cnt); // fixed dimension; - else obs_vars.obs_dim = add_dim(f_out, nc_dim_nobs); // unlimited dimension; + if (IS_INVALID_NC(obs_dim)) { + if (obs_cnt > 0) obs_dim = add_dim(f_out, nc_dim_nobs, obs_cnt); // fixed dimension; + else obs_dim = add_dim(f_out, nc_dim_nobs); // unlimited dimension; } - nc_data_buffer.obs_vars = obs_vars; } /////////////////////////////////////////////////////////////////////////////// -void create_nc_hdr_vars (NetcdfObsVars &obs_vars, NcFile *f_out, - const int hdr_count, const int deflate_level) { - const string method_name = " create_nc_hdr_vars()"; +void NetcdfObsVars::create_hdr_vars (NcFile *f_out, const int hdr_count) { + const string method_name = " create_hdr_vars()"; mlog << Debug(7) << method_name << " hdr_count: " << hdr_count << "\n"; - + // Define netCDF dimensions - create_nc_dimensions(obs_vars, f_out); - - NcDim hdr_dim; - obs_vars.hdr_cnt = hdr_count; - if (!IS_INVALID_NC(obs_vars.hdr_dim)) { - hdr_dim = obs_vars.hdr_dim; - } - else { + hdr_cnt = hdr_count; + create_dimensions(f_out); + + if (IS_INVALID_NC(hdr_dim)) { hdr_dim = (hdr_count > 0) ? add_dim(f_out, nc_dim_nhdr, hdr_count) : add_dim(f_out, nc_dim_nhdr); // unlimited dimension - obs_vars.hdr_dim = hdr_dim; } // Define netCDF header variables - obs_vars.hdr_typ_var = add_var(f_out, nc_var_hdr_typ, ncInt, hdr_dim, deflate_level); - obs_vars.hdr_sid_var = add_var(f_out, nc_var_hdr_sid, ncInt, hdr_dim, deflate_level); - obs_vars.hdr_vld_var = add_var(f_out, nc_var_hdr_vld, ncInt, hdr_dim, deflate_level); - obs_vars.hdr_lat_var = add_var(f_out, nc_var_hdr_lat, ncFloat, hdr_dim, deflate_level); - obs_vars.hdr_lon_var = add_var(f_out, nc_var_hdr_lon, ncFloat, hdr_dim, deflate_level); - obs_vars.hdr_elv_var = add_var(f_out, nc_var_hdr_elv, ncFloat, hdr_dim, deflate_level); - - add_att(&obs_vars.hdr_typ_var, "long_name", "index of message type"); - add_att(&obs_vars.hdr_typ_var, "_FillValue", (int)FILL_VALUE); - add_att(&obs_vars.hdr_sid_var, "long_name", "index of station identification"); - add_att(&obs_vars.hdr_sid_var, "_FillValue", (int)FILL_VALUE); - add_att(&obs_vars.hdr_vld_var, "long_name", "index of valid time"); - add_att(&obs_vars.hdr_vld_var, "_FillValue", (int)FILL_VALUE); - - add_att(&obs_vars.hdr_lat_var, "long_name", "latitude"); - add_att(&obs_vars.hdr_lat_var, "_FillValue", FILL_VALUE); - add_att(&obs_vars.hdr_lat_var, "units", "degrees_north"); - add_att(&obs_vars.hdr_lon_var, "long_name", "longitude"); - add_att(&obs_vars.hdr_lon_var, "_FillValue", FILL_VALUE); - add_att(&obs_vars.hdr_lon_var, "units", "degrees_east"); - add_att(&obs_vars.hdr_elv_var, "long_name", "elevation"); - add_att(&obs_vars.hdr_elv_var, "_FillValue", FILL_VALUE); - add_att(&obs_vars.hdr_elv_var, "units", "meters above sea level (msl)"); - nc_data_buffer.obs_vars = obs_vars; -} - -/////////////////////////////////////////////////////////////////////////////// - -NcDim create_nc_obs_var_var (NetcdfObsVars &obs_vars, NcFile *f_out, - int var_count, const int deflate_level) { - NcDim var_dim = add_dim(f_out, nc_dim_nvar, var_count); - // If the string length is modified, update nc_tools.cc, too. - if (IS_INVALID_NC(obs_vars.strl2_dim)) obs_vars.strl2_dim = add_dim(f_out, nc_dim_mxstr2, HEADER_STR_LEN2); - - obs_vars.obs_var = add_var(f_out, nc_var_obs_var, ncChar, var_dim, obs_vars.strl2_dim, deflate_level); - add_att(&obs_vars.obs_var, "long_name", "variable names"); - return var_dim; + hdr_typ_var = add_var(f_out, nc_var_hdr_typ, ncInt, hdr_dim, deflate_level); + hdr_sid_var = add_var(f_out, nc_var_hdr_sid, ncInt, hdr_dim, deflate_level); + hdr_vld_var = add_var(f_out, nc_var_hdr_vld, ncInt, hdr_dim, deflate_level); + hdr_lat_var = add_var(f_out, nc_var_hdr_lat, ncFloat, hdr_dim, deflate_level); + hdr_lon_var = add_var(f_out, nc_var_hdr_lon, ncFloat, hdr_dim, deflate_level); + hdr_elv_var = add_var(f_out, nc_var_hdr_elv, ncFloat, hdr_dim, deflate_level); + + add_att(&hdr_typ_var, "long_name", "index of message type"); + add_att(&hdr_typ_var, "_FillValue", (int)FILL_VALUE); + add_att(&hdr_sid_var, "long_name", "index of station identification"); + add_att(&hdr_sid_var, "_FillValue", (int)FILL_VALUE); + add_att(&hdr_vld_var, "long_name", "index of valid time"); + add_att(&hdr_vld_var, "_FillValue", (int)FILL_VALUE); + + add_att(&hdr_lat_var, "long_name", "latitude"); + add_att(&hdr_lat_var, "_FillValue", FILL_VALUE); + add_att(&hdr_lat_var, "units", "degrees_north"); + add_att(&hdr_lon_var, "long_name", "longitude"); + add_att(&hdr_lon_var, "_FillValue", FILL_VALUE); + add_att(&hdr_lon_var, "units", "degrees_east"); + add_att(&hdr_elv_var, "long_name", "elevation"); + add_att(&hdr_elv_var, "_FillValue", FILL_VALUE); + add_att(&hdr_elv_var, "units", "meters above sea level (msl)"); } /////////////////////////////////////////////////////////////////////////////// -void create_nc_obs_vars (NetcdfObsVars &obs_vars, NcFile *f_out, - const int deflate_level, bool use_var_id) { +void NetcdfObsVars::create_obs_vars (NcFile *f_out) { const char *long_name_str; - const string method_name = " create_nc_obs_vars()"; + const string method_name = " create_obs_vars()"; // Define netCDF dimensions - create_nc_dimensions(obs_vars, f_out); - + create_dimensions(f_out); + // Define netCDF variables - obs_vars.obs_qty_var = add_var(f_out, nc_var_obs_qty, ncInt, obs_vars.obs_dim, deflate_level); - obs_vars.obs_hid_var = add_var(f_out, nc_var_obs_hid, ncInt, obs_vars.obs_dim, deflate_level); + obs_qty_var = add_var(f_out, nc_var_obs_qty, ncInt, obs_dim, deflate_level); + obs_hid_var = add_var(f_out, nc_var_obs_hid, ncInt, obs_dim, deflate_level); if (use_var_id) - obs_vars.obs_vid_var = add_var(f_out, nc_var_obs_vid, ncInt, obs_vars.obs_dim, deflate_level); + obs_vid_var = add_var(f_out, nc_var_obs_vid, ncInt, obs_dim, deflate_level); else - obs_vars.obs_gc_var = add_var(f_out, nc_var_obs_gc, ncInt, obs_vars.obs_dim, deflate_level); - obs_vars.obs_lvl_var = add_var(f_out, nc_var_obs_lvl, ncFloat, obs_vars.obs_dim, deflate_level); - obs_vars.obs_hgt_var = add_var(f_out, nc_var_obs_hgt, ncFloat, obs_vars.obs_dim, deflate_level); - obs_vars.obs_val_var = add_var(f_out, nc_var_obs_val, ncFloat, obs_vars.obs_dim, deflate_level); + obs_gc_var = add_var(f_out, nc_var_obs_gc, ncInt, obs_dim, deflate_level); + obs_lvl_var = add_var(f_out, nc_var_obs_lvl, ncFloat, obs_dim, deflate_level); + obs_hgt_var = add_var(f_out, nc_var_obs_hgt, ncFloat, obs_dim, deflate_level); + obs_val_var = add_var(f_out, nc_var_obs_val, ncFloat, obs_dim, deflate_level); add_att(f_out, nc_att_obs_version, MET_NC_Obs_version); add_att(f_out, nc_att_use_var_id, (use_var_id ? "true" : "false")); // Add variable attributes - add_att(&obs_vars.obs_qty_var, "long_name", "index of quality flag"); - add_att(&obs_vars.obs_hid_var, "long_name", "index of matching header data"); - add_att(&obs_vars.obs_hid_var, "_FillValue", (int)FILL_VALUE); - add_att(&obs_vars.obs_val_var, "long_name", "observation value"); - add_att(&obs_vars.obs_val_var, "_FillValue", FILL_VALUE); + add_att(&obs_qty_var, "long_name", "index of quality flag"); + add_att(&obs_hid_var, "long_name", "index of matching header data"); + add_att(&obs_hid_var, "_FillValue", (int)FILL_VALUE); + add_att(&obs_val_var, "long_name", "observation value"); + add_att(&obs_val_var, "_FillValue", FILL_VALUE); if (use_var_id) { - long_name_str = (obs_vars.attr_pb2nc + long_name_str = (attr_pb2nc ? "index of BUFR variable corresponding to the observation type" : "index of variable names at var_name"); - add_att(&obs_vars.obs_vid_var, "long_name", long_name_str); - add_att(&obs_vars.obs_vid_var, "_FillValue", (int)FILL_VALUE); + add_att(&obs_vid_var, "long_name", long_name_str); + add_att(&obs_vid_var, "_FillValue", (int)FILL_VALUE); } else { - add_att(&obs_vars.obs_gc_var, "long_name", "grib code corresponding to the observation type"); - add_att(&obs_vars.obs_gc_var, "_FillValue", (int)FILL_VALUE); + add_att(&obs_gc_var, "long_name", "grib code corresponding to the observation type"); + add_att(&obs_gc_var, "_FillValue", (int)FILL_VALUE); } - add_att(&obs_vars.obs_lvl_var, "long_name", "pressure level (hPa) or accumulation interval (sec)"); - add_att(&obs_vars.obs_lvl_var, "_FillValue", FILL_VALUE); - long_name_str = (obs_vars.attr_agl) + add_att(&obs_lvl_var, "long_name", "pressure level (hPa) or accumulation interval (sec)"); + add_att(&obs_lvl_var, "_FillValue", FILL_VALUE); + long_name_str = (attr_agl) ? "height in meters above sea level or ground level (msl or agl)" : "height in meters above sea level (msl)"; - add_att(&obs_vars.obs_hgt_var, "long_name", long_name_str); - add_att(&obs_vars.obs_hgt_var, "_FillValue", FILL_VALUE); - nc_data_buffer.obs_vars = obs_vars; + add_att(&obs_hgt_var, "long_name", long_name_str); + add_att(&obs_hgt_var, "_FillValue", FILL_VALUE); } /////////////////////////////////////////////////////////////////////////////// -void create_nc_obs_name_vars (NetcdfObsVars &obs_vars, NcFile *f_out, - const int var_count, const int unit_count, const int deflate_level) { - const string method_name = " create_nc_other_vars()"; - +void NetcdfObsVars::create_obs_name_vars (NcFile *f_out, const int var_count, const int unit_count) { + const string method_name = " create_other_vars()"; + if (var_count > 0) { - NcDim var_dim = create_nc_obs_var_var(obs_vars, f_out, var_count, deflate_level); + NcDim var_dim = create_var_obs_var(f_out, var_count); if (unit_count > 0) { - obs_vars.unit_var = add_var(f_out, nc_var_unit, ncChar, var_dim, obs_vars.strl2_dim, deflate_level); - obs_vars.desc_var = add_var(f_out, nc_var_desc, ncChar, var_dim, obs_vars.strl3_dim, deflate_level); - - add_att(&obs_vars.unit_var, "long_name", "variable units"); - add_att(&obs_vars.desc_var, "long_name", "variable descriptions"); + unit_var = add_var(f_out, nc_var_unit, ncChar, var_dim, strl2_dim, deflate_level); + desc_var = add_var(f_out, nc_var_desc, ncChar, var_dim, strl3_dim, deflate_level); + + add_att(&unit_var, "long_name", "variable units"); + add_att(&desc_var, "long_name", "variable descriptions"); } } } +//////////////////////////////////////////////////////////////////////// + +void NetcdfObsVars::create_pb_hdrs (NcFile *f_out, const int hdr_count) { + const string method_name = " create_pb_hdrs()"; + mlog << Debug(7) << method_name << " hdr_count: " << hdr_count << "\n"; + + // Define netCDF dimensions + if (IS_INVALID_NC(pb_hdr_dim)) pb_hdr_dim = add_dim(f_out, nc_dim_npbhdr, hdr_count); + + raw_hdr_cnt = hdr_count; + hdr_prpt_typ_var = add_var(f_out, nc_var_hdr_prpt_typ, ncInt, pb_hdr_dim, deflate_level); + hdr_irpt_typ_var = add_var(f_out, nc_var_hdr_irpt_typ, ncInt, pb_hdr_dim, deflate_level); + hdr_inst_typ_var = add_var(f_out, nc_var_hdr_inst_typ, ncInt, pb_hdr_dim, deflate_level); + add_att(&hdr_prpt_typ_var, "long_name", "PB report type"); + add_att(&hdr_prpt_typ_var, "_FillValue", (int)FILL_VALUE); + add_att(&hdr_irpt_typ_var, "long_name", "In report type"); + add_att(&hdr_irpt_typ_var, "_FillValue", (int)FILL_VALUE); + add_att(&hdr_inst_typ_var, "long_name", "instrument type"); + add_att(&hdr_inst_typ_var, "_FillValue", (int)FILL_VALUE); +} + /////////////////////////////////////////////////////////////////////////////// -void create_nc_table_vars (NetcdfObsVars &obs_vars, NcFile *f_out, const int deflate_level) { - const string method_name = " create_nc_table_vars()"; - +void NetcdfObsVars::create_table_vars (NcFile *f_out, NcHeaderData &hdr_data, + NcDataBuffer &data_buffer) { + const string method_name = " create_table_vars()"; + // Define netCDF dimensions NcDim hdr_typ_dim = add_dim(f_out, nc_dim_nhdr_typ, hdr_data.typ_array.n_elements()); NcDim hdr_sid_dim = add_dim(f_out, nc_dim_nhdr_sid, hdr_data.sid_array.n_elements()); NcDim hdr_vld_dim = add_dim(f_out, nc_dim_nhdr_vld, hdr_data.vld_array.n_elements()); - NcDim obs_qty_dim = add_dim(f_out, nc_dim_nqty, nc_data_buffer.qty_data_array.n_elements()); + NcDim obs_qty_dim = add_dim(f_out, nc_dim_nqty, data_buffer.qty_data_array.n_elements()); // Define netCDF header variables - obs_vars.hdr_typ_tbl_var = add_var(f_out, nc_var_hdr_typ_tbl, ncChar, hdr_typ_dim, obs_vars.strl2_dim, deflate_level); - obs_vars.hdr_sid_tbl_var = add_var(f_out, nc_var_hdr_sid_tbl, ncChar, hdr_sid_dim, obs_vars.strl2_dim, deflate_level); - obs_vars.hdr_vld_tbl_var = add_var(f_out, nc_var_hdr_vld_tbl, ncChar, hdr_vld_dim, obs_vars.strl_dim, deflate_level); - obs_vars.obs_qty_tbl_var = add_var(f_out, nc_var_obs_qty_tbl, ncChar, obs_qty_dim, obs_vars.strl_dim, deflate_level); + hdr_typ_tbl_var = add_var(f_out, nc_var_hdr_typ_tbl, ncChar, hdr_typ_dim, strl2_dim, deflate_level); + hdr_sid_tbl_var = add_var(f_out, nc_var_hdr_sid_tbl, ncChar, hdr_sid_dim, strl2_dim, deflate_level); + hdr_vld_tbl_var = add_var(f_out, nc_var_hdr_vld_tbl, ncChar, hdr_vld_dim, strl_dim, deflate_level); + obs_qty_tbl_var = add_var(f_out, nc_var_obs_qty_tbl, ncChar, obs_qty_dim, strl_dim, deflate_level); + + add_att(&obs_qty_tbl_var, "long_name", "quality flag"); + add_att(&hdr_typ_tbl_var, "long_name", "message type"); + add_att(&hdr_sid_tbl_var, "long_name", "station identification"); + add_att(&hdr_vld_tbl_var, "long_name", "valid time"); + add_att(&hdr_vld_tbl_var, "units", "YYYYMMDD_HHMMSS UTC"); +} - add_att(&obs_vars.obs_qty_tbl_var, "long_name", "quality flag"); - add_att(&obs_vars.hdr_typ_tbl_var, "long_name", "message type"); - add_att(&obs_vars.hdr_sid_tbl_var, "long_name", "station identification"); - add_att(&obs_vars.hdr_vld_tbl_var, "long_name", "valid time"); - add_att(&obs_vars.hdr_vld_tbl_var, "units", "YYYYMMDD_HHMMSS UTC"); +/////////////////////////////////////////////////////////////////////////////// + +NcDim NetcdfObsVars::create_var_obs_var (NcFile *f_out, int var_count) { + NcDim var_dim = add_dim(f_out, nc_dim_nvar, var_count); + // If the string length is modified, update nc_tools.cc, too. + if (IS_INVALID_NC(strl2_dim)) strl2_dim = add_dim(f_out, nc_dim_mxstr2, HEADER_STR_LEN2); + obs_var = add_var(f_out, nc_var_obs_var, ncChar, var_dim, strl2_dim, deflate_level); + add_att(&obs_var, "long_name", "variable names"); + return var_dim; } //////////////////////////////////////////////////////////////////////// -void create_nc_pb_hdrs (NetcdfObsVars &obs_vars, NcFile *f_out, - const int hdr_count, const int deflate_level) { - const string method_name = " create_nc_pb_hdrs()"; - mlog << Debug(7) << method_name << " hdr_count: " << hdr_count << "\n"; - - // Define netCDF dimensions - if (IS_INVALID_NC(obs_vars.pb_hdr_dim)) obs_vars.pb_hdr_dim = add_dim(f_out, nc_dim_npbhdr, hdr_count); - - obs_vars.raw_hdr_cnt = hdr_count; - obs_vars.hdr_prpt_typ_var = add_var(f_out, nc_var_hdr_prpt_typ, ncInt, obs_vars.pb_hdr_dim, deflate_level); - obs_vars.hdr_irpt_typ_var = add_var(f_out, nc_var_hdr_irpt_typ, ncInt, obs_vars.pb_hdr_dim, deflate_level); - obs_vars.hdr_inst_typ_var = add_var(f_out, nc_var_hdr_inst_typ, ncInt, obs_vars.pb_hdr_dim, deflate_level); - add_att(&obs_vars.hdr_prpt_typ_var, "long_name", "PB report type"); - add_att(&obs_vars.hdr_prpt_typ_var, "_FillValue", (int)FILL_VALUE); - add_att(&obs_vars.hdr_irpt_typ_var, "long_name", "In report type"); - add_att(&obs_vars.hdr_irpt_typ_var, "_FillValue", (int)FILL_VALUE); - add_att(&obs_vars.hdr_inst_typ_var, "long_name", "instrument type"); - add_att(&obs_vars.hdr_inst_typ_var, "_FillValue", (int)FILL_VALUE); -} +bool NetcdfObsVars::is_valid(bool exit_on_error) { + bool valid = true; + const char* method_name = "NetcdfObsVars::is_valid()"; + StringArray missing_dims; + StringArray missing_vars; -//////////////////////////////////////////////////////////////////////// + if(IS_INVALID_NC(hdr_dim)) missing_dims.add(nc_dim_nhdr); + if(IS_INVALID_NC(obs_dim)) missing_dims.add(nc_dim_nobs); + if(IS_INVALID_NC(strl_dim)) missing_dims.add("mxstr"); + + if(IS_INVALID_NC(hdr_typ_var)) missing_vars.add("hdr_typ"); + if(IS_INVALID_NC(hdr_sid_var)) missing_vars.add("hdr_sid"); + if(IS_INVALID_NC(hdr_vld_var)) missing_vars.add("hdr_vld"); + if(IS_INVALID_NC(hdr_arr_var) && IS_INVALID_NC(hdr_lat_var)) missing_vars.add("hdr_lat/hdr_arr"); + if(IS_INVALID_NC(obs_arr_var) && IS_INVALID_NC(obs_val_var)) missing_vars.add("obs_val/obs_arr"); + + if (0 < missing_dims.n()) { + valid = false; + for (int idx=0; idx " + << "can not find the dimension\"" << missing_dims[idx] << "\".\n\n"; + } + } + + if (0 < missing_vars.n()) { + valid = false; + for (int idx=0; idx " + << "can't read \"" << missing_vars[idx] + << "\" variable from the netCDF file\n\n"; + } + } -NcDataBuffer *get_nc_data_buffer() { - return &nc_data_buffer; + if (!valid && exit_on_error) exit(1); + + if(IS_INVALID_NC(obs_qty_var)) + mlog << Debug(3) << "Quality marker information not found input file.\n"; + + return valid; } -//////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +void NetcdfObsVars::read_dims_vars(NcFile *f_in) { + + NcVar ncVar; + // Define netCDF dimensions + strl_dim = get_nc_dim(f_in, nc_dim_mxstr); // header string dimension + if (has_dim(f_in, nc_dim_mxstr2)) + strl2_dim = get_nc_dim(f_in, nc_dim_mxstr2); // header string dimension (bigger dimension) + if (has_dim(f_in, nc_dim_mxstr3)) + strl3_dim = get_nc_dim(f_in, nc_dim_mxstr3); // header string dimension (bigger dimension) + + if (has_dim(f_in, nc_dim_hdr_arr)) { + hdr_arr_dim = get_nc_dim(f_in, nc_dim_hdr_arr); // Header array width + obs_arr_dim = get_nc_dim(f_in, nc_dim_obs_arr); // Observation array width + } + else { + if (has_dim(f_in, nc_dim_nhdr_typ)) + hdr_typ_dim = get_nc_dim(f_in, nc_dim_nhdr_typ); // header dimension for message type + if (has_dim(f_in, nc_dim_nhdr_sid)) + hdr_sid_dim = get_nc_dim(f_in, nc_dim_nhdr_sid); // header dimension for station id + if (has_dim(f_in, nc_dim_nhdr_vld)) + hdr_vld_dim = get_nc_dim(f_in, nc_dim_nhdr_vld); // header dimension for valid time + if (has_dim(f_in, nc_dim_npbhdr)) + pb_hdr_dim = get_nc_dim(f_in, nc_dim_npbhdr); // header dimension for PB headers + } + obs_dim = get_nc_dim(f_in, nc_dim_nobs); // Observation array length + hdr_dim = get_nc_dim(f_in, nc_dim_nhdr); // Header array length + + // Get netCDF header variables + hdr_typ_var = get_var(f_in, nc_var_hdr_typ); // Message type (String or int) + hdr_sid_var = get_var(f_in, nc_var_hdr_sid); // Station ID (String or int) + hdr_vld_var = get_var(f_in, nc_var_hdr_vld); // Valid time (String or int) + + ncVar = get_var(f_in, nc_var_hdr_lat); + if (IS_INVALID_NC(ncVar)) { + hdr_arr_var = get_var(f_in, nc_var_hdr_arr); // Header array + } else { + hdr_lat_var = ncVar; // Header array + hdr_lon_var = get_var(f_in, nc_var_hdr_lon); // Header array + hdr_elv_var = get_var(f_in, nc_var_hdr_elv); // Header array + hdr_typ_tbl_var = get_var(f_in, nc_var_hdr_typ_tbl); // Message type (String) + hdr_sid_tbl_var = get_var(f_in, nc_var_hdr_sid_tbl); // Station ID (String) + hdr_vld_tbl_var = get_var(f_in, nc_var_hdr_vld_tbl); // Valid time (String) + } + + // Get netCDF variables + ncVar = get_var(f_in, nc_var_obs_hid); + if (IS_INVALID_NC(ncVar)) { + obs_arr_var = get_var(f_in, nc_var_obs_arr); + } else { + obs_hid_var = ncVar; // Obs. header id array + ncVar = get_var(f_in, nc_var_obs_gc); + if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array + ncVar = get_var(f_in, nc_var_obs_vid); + if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array + obs_lvl_var = get_var(f_in, nc_var_obs_lvl); // Obs. pressure level array + obs_hgt_var = get_var(f_in, nc_var_obs_hgt); // Obs. highth array + obs_val_var = get_var(f_in, nc_var_obs_val); // Obs. value array + } + ncVar = get_var(f_in, nc_var_obs_qty); + if (!IS_INVALID_NC(ncVar)) obs_qty_var = ncVar; + ncVar = get_var(f_in, nc_var_obs_qty_tbl); + if (!IS_INVALID_NC(ncVar)) obs_qty_tbl_var = ncVar; -NcHeaderData *get_hdr_data_buffer() { - return &hdr_data; + ncVar = get_var(f_in, nc_var_obs_var); + if (!IS_INVALID_NC(ncVar)) obs_var = ncVar; + ncVar = get_var(f_in, nc_var_unit); + if (!IS_INVALID_NC(ncVar)) unit_var = ncVar; + ncVar = get_var(f_in, nc_var_desc); + if (!IS_INVALID_NC(ncVar)) desc_var = ncVar; + + // PrepBufr only headers + ncVar = get_var(f_in, nc_var_hdr_prpt_typ); + if (!IS_INVALID_NC(ncVar)) hdr_prpt_typ_var = ncVar; + ncVar = get_var(f_in, nc_var_hdr_irpt_typ); + if (!IS_INVALID_NC(ncVar)) hdr_irpt_typ_var = ncVar; + ncVar = get_var(f_in, nc_var_hdr_inst_typ); + if (!IS_INVALID_NC(ncVar)) hdr_inst_typ_var = ncVar; + + bool _use_var_id = false; + if (!get_global_att(f_in, nc_att_use_var_id, _use_var_id)) { + _use_var_id = IS_VALID_NC(obs_var); + } + + use_var_id = _use_var_id; } //////////////////////////////////////////////////////////////////////// -NcHeaderData get_nc_hdr_data(NetcdfObsVars obs_vars) { - NcHeaderData my_hdr_data; - long nhdr_count = get_dim_size(&obs_vars.hdr_dim); - int strl_len = get_dim_size(&obs_vars.strl_dim); +void NetcdfObsVars::read_header_data(NcHeaderData &hdr_data) { + bool is_valid_obs_nc = true; + long nhdr_count = get_dim_size(&hdr_dim); + int strl_len = get_dim_size(&strl_dim); int strl2_len = strl_len; int typ_len = strl_len; int sid_len = strl_len; int vld_len = strl_len; - int hdr_arr_len = IS_INVALID_NC(obs_vars.hdr_arr_dim) - ? 0 : get_dim_size(&obs_vars.hdr_arr_dim); - bool has_array_vars = IS_INVALID_NC(obs_vars.hdr_typ_tbl_var); + int hdr_arr_len = IS_INVALID_NC(hdr_arr_dim) + ? 0 : get_dim_size(&hdr_arr_dim); + bool has_array_vars = IS_INVALID_NC(hdr_typ_tbl_var); + const char *method_name = "get_nc_header() -> "; - if (!IS_INVALID_NC(obs_vars.strl2_dim)) { + if (!IS_INVALID_NC(strl2_dim)) { NcDim str_dim; - strl2_len = get_dim_size(&obs_vars.strl2_dim); - string dim_name = GET_NC_NAME(obs_vars.strl2_dim); - str_dim = get_nc_dim((IS_INVALID_NC(obs_vars.hdr_typ_tbl_var) - ? &obs_vars.hdr_typ_var : &obs_vars.hdr_typ_tbl_var), dim_name); + strl2_len = get_dim_size(&strl2_dim); + string dim_name = GET_NC_NAME(strl2_dim); + str_dim = get_nc_dim((IS_INVALID_NC(hdr_typ_tbl_var) + ? &hdr_typ_var : &hdr_typ_tbl_var), dim_name); if (!IS_INVALID_NC(str_dim)) typ_len = strl2_len; - - str_dim = get_nc_dim((IS_INVALID_NC(obs_vars.hdr_sid_tbl_var) - ? &obs_vars.hdr_sid_var : &obs_vars.hdr_sid_tbl_var), dim_name); + + str_dim = get_nc_dim((IS_INVALID_NC(hdr_sid_tbl_var) + ? &hdr_sid_var : &hdr_sid_tbl_var), dim_name); if (!IS_INVALID_NC(str_dim)) sid_len = strl2_len; - - str_dim = get_nc_dim((IS_INVALID_NC(obs_vars.hdr_vld_tbl_var) - ? &obs_vars.hdr_vld_var : &obs_vars.hdr_vld_tbl_var), dim_name); + + str_dim = get_nc_dim((IS_INVALID_NC(hdr_vld_tbl_var) + ? &hdr_vld_var : &hdr_vld_tbl_var), dim_name); if (!IS_INVALID_NC(str_dim)) vld_len = strl2_len; } - - my_hdr_data.typ_len = typ_len; - my_hdr_data.sid_len = sid_len; - my_hdr_data.vld_len = vld_len; - my_hdr_data.strl_len = strl_len; - my_hdr_data.strll_len = strl2_len; - + + hdr_data.typ_len = typ_len; + hdr_data.sid_len = sid_len; + hdr_data.vld_len = vld_len; + hdr_data.strl_len = strl_len; + hdr_data.strll_len = strl2_len; + + hdr_data.lat_array.extend(nhdr_count); + hdr_data.lon_array.extend(nhdr_count); + hdr_data.elv_array.extend(nhdr_count); + hdr_data.typ_idx_array.extend(nhdr_count); + hdr_data.sid_idx_array.extend(nhdr_count); + hdr_data.vld_idx_array.extend(nhdr_count); + int buf_size = ((nhdr_count > NC_BUFFER_SIZE_32K) - ? NC_BUFFER_SIZE_32K : (nhdr_count)); - + ? NC_BUFFER_SIZE_32K : (nhdr_count)); + // // Allocate space to store the data // @@ -462,124 +666,120 @@ NcHeaderData get_nc_hdr_data(NetcdfObsVars obs_vars) { long lengths[2] = { 1, 1 }; long offsets_1D[1] = { 0 }; long lengths_1D[1] = { 1 }; - - //lengths[0] = buf_size; - //lengths[1] = strl_len; + for(int i_start=0; i_start NC_BUFFER_SIZE_32K) ? NC_BUFFER_SIZE_32K : (nhdr_count-i_start); - - offsets[0] = i_start; - lengths[0] = buf_size; - offsets_1D[0] = i_start; - lengths_1D[0] = buf_size; - + + offsets[0] = offsets_1D[0] = i_start; + lengths[0] = lengths_1D[0] = buf_size; + // // Get the corresponding header message type // if (has_array_vars) { lengths[1] = typ_len; - if(!get_nc_data(&obs_vars.hdr_typ_var, + if(!get_nc_data(&hdr_typ_var, (char *)&hdr_typ_block[0], lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_typ\n\n"; exit(1); } - + // // Get the corresponding header station id // lengths[1] = sid_len; - if(!get_nc_data(&obs_vars.hdr_sid_var, + if(!get_nc_data(&hdr_sid_var, (char *)&hdr_sid_block[0], lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_sid\n\n"; exit(1); } - + // // Get the corresponding header valid time // lengths[1] = vld_len; - if(!get_nc_data(&obs_vars.hdr_vld_var, + if(!get_nc_data(&hdr_vld_var, (char *)&hdr_vld_block[0], lengths, offsets)) { mlog << Error << "\nget_nc_header() -> " << "trouble getting hdr_vld\n\n"; exit(1); } - + // // Get the header for this observation // lengths[1] = hdr_arr_len; - if(!get_nc_data(&obs_vars.hdr_arr_var, + if(!get_nc_data(&hdr_arr_var, (float *)&hdr_arr_block[0], lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_arr\n\n"; exit(1); } for (int hIndex = 0; hIndex < buf_size; hIndex++) { - my_hdr_data.typ_array.add(hdr_typ_block[hIndex]); - my_hdr_data.sid_array.add(hdr_sid_block[hIndex]); - my_hdr_data.vld_array.add(hdr_vld_block[hIndex]); - my_hdr_data.lat_array.add(hdr_arr_block[hIndex][0]); - my_hdr_data.lon_array.add(hdr_arr_block[hIndex][1]); - my_hdr_data.elv_array.add(hdr_arr_block[hIndex][2]); + hdr_data.typ_array.add(hdr_typ_block[hIndex]); + hdr_data.sid_array.add(hdr_sid_block[hIndex]); + hdr_data.vld_array.add(hdr_vld_block[hIndex]); + hdr_data.lat_array.add(hdr_arr_block[hIndex][0]); + hdr_data.lon_array.add(hdr_arr_block[hIndex][1]); + hdr_data.elv_array.add(hdr_arr_block[hIndex][2]); } } else { // Get the corresponding header message type (index, not string) - if(!get_nc_data(&obs_vars.hdr_typ_var, + if(!get_nc_data(&hdr_typ_var, hdr_typ_idx_block, lengths_1D, offsets_1D)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_typ\n\n"; exit(1); } - + // Get the corresponding header station id (index, not string) - if(!get_nc_data(&obs_vars.hdr_sid_var, + if(!get_nc_data(&hdr_sid_var, hdr_sid_idx_block, lengths_1D, offsets_1D)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_sid\n\n"; exit(1); } - + // Get the corresponding header valid time (index, not string) - if(!get_nc_data(&obs_vars.hdr_vld_var, + if(!get_nc_data(&hdr_vld_var, hdr_vld_idx_block, lengths_1D, offsets_1D)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_vld\n\n"; exit(1); } - + // // Get the header for this observation // - if(!get_nc_data(&obs_vars.hdr_lat_var, + if(!get_nc_data(&hdr_lat_var, hdr_lat_block, lengths_1D, offsets_1D)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_lat\n\n"; exit(1); } - if(!get_nc_data(&obs_vars.hdr_lon_var, + if(!get_nc_data(&hdr_lon_var, hdr_lon_block, lengths_1D, offsets_1D)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_lon\n\n"; exit(1); } - if(!get_nc_data(&obs_vars.hdr_elv_var, + if(!get_nc_data(&hdr_elv_var, hdr_elv_block, lengths_1D, offsets_1D)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_elv\n\n"; exit(1); } for (int hIndex = 0; hIndex < buf_size; hIndex++) { - my_hdr_data.typ_idx_array.add(hdr_typ_idx_block[hIndex]); - my_hdr_data.sid_idx_array.add(hdr_sid_idx_block[hIndex]); - my_hdr_data.vld_idx_array.add(hdr_vld_idx_block[hIndex]); - my_hdr_data.lat_array.add(hdr_lat_block[hIndex]); - my_hdr_data.lon_array.add(hdr_lon_block[hIndex]); - my_hdr_data.elv_array.add(hdr_elv_block[hIndex]); + hdr_data.typ_idx_array.add(hdr_typ_idx_block[hIndex]); + hdr_data.sid_idx_array.add(hdr_sid_idx_block[hIndex]); + hdr_data.vld_idx_array.add(hdr_vld_idx_block[hIndex]); + hdr_data.lat_array.add(hdr_lat_block[hIndex]); + hdr_data.lon_array.add(hdr_lon_block[hIndex]); + hdr_data.elv_array.add(hdr_elv_block[hIndex]); } } } @@ -590,224 +790,115 @@ NcHeaderData get_nc_hdr_data(NetcdfObsVars obs_vars) { delete[] hdr_lat_block; delete[] hdr_lon_block; delete[] hdr_elv_block; - + if (!has_array_vars) { int tmp_dim_size; - + lengths[1] = typ_len; - tmp_dim_size = get_dim_size(&obs_vars.hdr_typ_dim); + tmp_dim_size = get_dim_size(&hdr_typ_dim); for(int i_start=0; i_start NC_BUFFER_SIZE_32K) ? NC_BUFFER_SIZE_32K : (tmp_dim_size-i_start); offsets[0] = i_start; lengths[0] = buf_size; - + // Get the corresponding header message type (string) - if(!get_nc_data(&obs_vars.hdr_typ_tbl_var, + if(!get_nc_data(&hdr_typ_tbl_var, (char *)&hdr_typ_block[0], lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_typ\n\n"; exit(1); } for (int hIndex = 0; hIndex < buf_size; hIndex++) { - my_hdr_data.typ_array.add(hdr_typ_block[hIndex]); + hdr_data.typ_array.add(hdr_typ_block[hIndex]); } } - + lengths[1] = sid_len; - tmp_dim_size = get_dim_size(&obs_vars.hdr_sid_dim); + tmp_dim_size = get_dim_size(&hdr_sid_dim); for(int i_start=0; i_start NC_BUFFER_SIZE_32K) ? NC_BUFFER_SIZE_32K : (tmp_dim_size-i_start); offsets[0] = i_start; lengths[0] = buf_size; - + // Get the corresponding header station id (string) - if(!get_nc_data(&obs_vars.hdr_sid_tbl_var, + if(!get_nc_data(&hdr_sid_tbl_var, (char *)&hdr_sid_block[0], lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_typ\n\n"; exit(1); } for (int hIndex = 0; hIndex < buf_size; hIndex++) { - my_hdr_data.sid_array.add(hdr_sid_block[hIndex]); + hdr_data.sid_array.add(hdr_sid_block[hIndex]); } } lengths[1] = vld_len; - tmp_dim_size = get_dim_size(&obs_vars.hdr_vld_dim); + tmp_dim_size = get_dim_size(&hdr_vld_dim); for(int i_start=0; i_start NC_BUFFER_SIZE_32K) ? NC_BUFFER_SIZE_32K : (tmp_dim_size-i_start); offsets[0] = i_start; lengths[0] = buf_size; - + // Get the corresponding header valid time (string) - if(!get_nc_data(&obs_vars.hdr_vld_tbl_var, + if(!get_nc_data(&hdr_vld_tbl_var, (char *)&hdr_vld_block[0], lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " + mlog << Error << "\n" << method_name << "trouble getting hdr_typ\n\n"; exit(1); } for (int hIndex = 0; hIndex < buf_size; hIndex++) { - my_hdr_data.vld_array.add(hdr_vld_block[hIndex]); + hdr_data.vld_array.add(hdr_vld_block[hIndex]); } } } - if (!IS_INVALID_NC(obs_vars.pb_hdr_dim)) { - get_nc_pb_hdr_data(obs_vars, &my_hdr_data); - int last_prpt_typ = my_hdr_data.prpt_typ_array.n_elements() - 1; - int last_irpt_typ = my_hdr_data.irpt_typ_array.n_elements() - 1; - int last_inst_typ = my_hdr_data.inst_typ_array.n_elements() - 1; - mlog << Debug(10) - << " prpt_typ[0,-1] " << my_hdr_data.prpt_typ_array[0] - << "," << my_hdr_data.prpt_typ_array[last_prpt_typ] - << " irpt_typ[0,-1] " << my_hdr_data.irpt_typ_array[0] - << "," << my_hdr_data.irpt_typ_array[last_irpt_typ] - << " inst_typ[0,-1] " << my_hdr_data.inst_typ_array[0] - << "," << my_hdr_data.inst_typ_array[last_inst_typ] << "\n"; - } - return my_hdr_data; -} - -//////////////////////////////////////////////////////////////////////// -int get_nc_hdr_cur_index() { - //return (nc_data_buffer.cur_hdr_idx < 0) ? 0 : nc_data_buffer.cur_hdr_idx; - return nc_data_buffer.cur_hdr_idx; + if (!IS_INVALID_NC(pb_hdr_dim)) { + read_pb_hdr_data(hdr_data); + int last_prpt_typ = hdr_data.prpt_typ_array.n_elements() - 1; + int last_irpt_typ = hdr_data.irpt_typ_array.n_elements() - 1; + int last_inst_typ = hdr_data.inst_typ_array.n_elements() - 1; + if (0 > last_prpt_typ) mlog << Debug(7) << " prpt_typ is empty\n"; + else if (0 > last_irpt_typ) mlog << Debug(7) << " irpt_typ is empty\n"; + else if (0 > last_inst_typ) mlog << Debug(7) << " inst_typ is empty\n"; + else { + mlog << Debug(10) + << " prpt_typ[0,-1] " << hdr_data.prpt_typ_array[0] + << "," << hdr_data.prpt_typ_array[last_prpt_typ] + << " irpt_typ[0,-1] " << hdr_data.irpt_typ_array[0] + << "," << hdr_data.irpt_typ_array[last_irpt_typ] + << " inst_typ[0,-1] " << hdr_data.inst_typ_array[0] + << "," << hdr_data.inst_typ_array[last_inst_typ] << "\n"; + } + } } //////////////////////////////////////////////////////////////////////// -int get_nc_obs_buf_index() { - return nc_data_buffer.obs_data_idx; -} - -//////////////////////////////////////////////////////////////////////// +bool NetcdfObsVars::read_obs_data(int buf_size, int offset, + int qty_len, float *obs_arr, int *qty_idx_arr, char *obs_qty_buf) { + bool result = true; + long offsets[2] = { offset, 0 }; + long lengths[2] = { buf_size, 1 }; + const char *method_name = "read_obs_data() -> "; -void get_nc_pb_hdr_data(NetcdfObsVars obs_vars, NcHeaderData *my_hdr_data) { + if (IS_VALID_NC(obs_arr_var)) { + // Read the current observation message + lengths[1] = OBS_ARRAY_LEN; + if(!get_nc_data(&obs_arr_var, (float *)obs_arr, lengths, offsets)) { + mlog << Error << "\n" << method_name << "trouble getting obs_arr\n\n"; + result = false; + } - long offsets[1] = { 0 }; - long lengths[1] = { 1 }; - int pb_hdr_count = get_dim_size(&obs_vars.pb_hdr_dim); - - // Read PB report type - int buf_size = ((pb_hdr_count > NC_BUFFER_SIZE_32K) - ? NC_BUFFER_SIZE_32K : (pb_hdr_count)); - int *hdr_prpt_typ_block = new int[buf_size]; - int *hdr_irpt_typ_block = new int[buf_size]; - int *hdr_inst_typ_block = new int[buf_size]; - for(int i_start=0; i_start NC_BUFFER_SIZE_32K) buf_size = NC_BUFFER_SIZE_32K; - offsets[0] = i_start; - lengths[0] = buf_size; - - if (!IS_INVALID_NC(obs_vars.hdr_prpt_typ_var)) { - // Get the corresponding header PB message type (string) - if(!get_nc_data(&obs_vars.hdr_prpt_typ_var, - hdr_prpt_typ_block, lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " - << "trouble getting hdr_prpt_typ\n\n"; - exit(1); + if (0 != obs_qty_buf) { + lengths[1] = qty_len; + if(!get_nc_data(&obs_qty_var, obs_qty_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << "trouble getting obs_qty\n\n"; + result = false; } } - - if (!IS_INVALID_NC(obs_vars.hdr_irpt_typ_var)) { - // Get the corresponding header In message type (string) - if(!get_nc_data(&obs_vars.hdr_irpt_typ_var, - hdr_irpt_typ_block, lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " - << "trouble getting hdr_irpt_typ\n\n"; - exit(1); - } - } - - if (!IS_INVALID_NC(obs_vars.hdr_inst_typ_var)) { - // Get the corresponding header instrument type (string) - if(!get_nc_data(&obs_vars.hdr_inst_typ_var, - hdr_inst_typ_block, lengths, offsets)) { - mlog << Error << "\nget_nc_header() -> " - << "trouble getting hdr_inst_typ\n\n"; - exit(1); - } - } - - for (int hIndex = 0; hIndex < buf_size; hIndex++) { - my_hdr_data->prpt_typ_array.add(hdr_prpt_typ_block[hIndex]); - my_hdr_data->irpt_typ_array.add(hdr_irpt_typ_block[hIndex]); - my_hdr_data->inst_typ_array.add(hdr_inst_typ_block[hIndex]); - } - } - - delete[] hdr_prpt_typ_block; - delete[] hdr_irpt_typ_block; - delete[] hdr_inst_typ_block; - -} - -/////////////////////////////////////////////////////////////////////////////// - -void init_nc_dims_vars_config(NetcdfObsVars &obs_vars, bool use_var_id) { - obs_vars.attr_agl = false; - obs_vars.attr_pb2nc = false; - obs_vars.use_var_id = use_var_id; - obs_vars.hdr_cnt = 0; // header array length (fixed dimension if hdr_cnt > 0) - obs_vars.obs_cnt = 0; // obs. array length (fixed dimension if obs_cnt > 0) - //obs_vars.hdr_str_len = 0; // string length for header (message) type header - nc_data_buffer.obs_vars = obs_vars; -} - -//////////////////////////////////////////////////////////////////////// - -bool is_using_var_id(const char * nc_name) { - bool use_var_id = false; - ConcatString attr_name = nc_att_use_var_id; - if (!get_global_att(nc_name, nc_att_use_var_id, use_var_id)) { - use_var_id = false; - } - return use_var_id; -} - -/////////////////////////////////////////////////////////////////////////////// - -void nc_obs_initialize() { - nc_data_buffer.obs_data_idx = 0; - nc_data_buffer.obs_data_offset = 0; - nc_data_buffer.hdr_data_idx = 0; - nc_data_buffer.hdr_data_offset = 0; - nc_data_buffer.pb_hdr_data_offset = 0; - - strcpy(nc_data_buffer.prev_hdr_typ_buf, "NotDefined"); - strcpy(nc_data_buffer.prev_hdr_sid_buf, "NotDefined"); - strcpy(nc_data_buffer.prev_hdr_vld_buf, "NotDefined"); - for (int index=0; index trouble getting obs_arr\n\n"; - result = false; - } - - lengths[1] = qty_len; - if(!get_nc_data(&obs_vars.obs_qty_var, obs_qty_buf, lengths, offsets)) { - mlog << Error << "\nprocess_point_obs() -> trouble getting obs_qty\n\n"; - result = false; - } } else { int *obs_hid_buf = new int[buf_size]; @@ -815,48 +906,48 @@ bool read_nc_obs_data(NetcdfObsVars obs_vars, int buf_size, int offset, float *obs_lvl_buf = new float[buf_size]; float *obs_hgt_buf = new float[buf_size]; float *obs_val_buf = new float[buf_size]; - + lengths[1] = 1; - - if(!get_nc_data(&obs_vars.obs_hid_var, obs_hid_buf, lengths, offsets)) { - mlog << Error << "\nread_nc_obs_array() -> " + + if(!get_nc_data(&obs_hid_var, obs_hid_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << "can't read the record for observation " << "index " << offset << "\n\n"; result = false; } - if(!get_nc_data((IS_INVALID_NC(obs_vars.obs_gc_var) ? &obs_vars.obs_vid_var : &obs_vars.obs_gc_var), + if(!get_nc_data((IS_INVALID_NC(obs_gc_var) ? &obs_vid_var : &obs_gc_var), obs_vid_buf, lengths, offsets)) { - mlog << Error << "\nread_nc_obs_array() -> " + mlog << Error << "\n" << method_name << "can't read the record (vid or gc) for observation " << "index " << offset << "\n\n"; result = false; } - if(!get_nc_data(&obs_vars.obs_lvl_var, obs_lvl_buf, lengths, offsets)) { - mlog << Error << "\nread_nc_obs_array() -> " + if(!get_nc_data(&obs_lvl_var, obs_lvl_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << "can't read the record (lvl) for observation " << "index " << offset << "\n\n"; result = false; } - if(!get_nc_data(&obs_vars.obs_hgt_var, obs_hgt_buf, lengths, offsets)) { - mlog << Error << "\nread_nc_obs_array() -> " + if(!get_nc_data(&obs_hgt_var, obs_hgt_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << "can't read the record (hgt) for observation " << "index " << offset << "\n\n"; result = false; } - if(!get_nc_data(&obs_vars.obs_val_var, obs_val_buf, lengths, offsets)) { - mlog << Error << "\nread_nc_obs_array() -> " + if(!get_nc_data(&obs_val_var, obs_val_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << "can't read the record (val) for observation " << "index " << offset << "\n\n"; result = false; } - if (!get_nc_data(&obs_vars.obs_qty_var, qty_idx_arr, lengths, offsets)) { - mlog << Error << "\nread_nc_obs_array() -> " + if (!get_nc_data(&obs_qty_var, qty_idx_arr, lengths, offsets)) { + mlog << Error << "\n" << method_name << "can't read the index of quality flag for observation " << "index " << offset << "\n\n"; result = false; } - + if (result) { float *tmp_obs_arr = obs_arr; for(int index=0; index 0) - obs_vars.strl_dim = get_nc_dim(f_in, nc_dim_mxstr); // header string dimension - if (has_dim(f_in, nc_dim_mxstr2)) - obs_vars.strl2_dim = get_nc_dim(f_in, nc_dim_mxstr2); // header string dimension (bigger dimension) - if (has_dim(f_in, nc_dim_mxstr3)) - obs_vars.strl3_dim = get_nc_dim(f_in, nc_dim_mxstr3); // header string dimension (bigger dimension) +void NetcdfObsVars::read_pb_hdr_data(NcHeaderData &hdr_data) { + const char *method_name = "get_nc_pb_hdr_data() -> "; - if (has_dim(f_in, nc_dim_hdr_arr)) { - obs_vars.hdr_arr_dim = get_nc_dim(f_in, nc_dim_hdr_arr); // Header array width - obs_vars.obs_arr_dim = get_nc_dim(f_in, nc_dim_obs_arr); // Observation array width + int pb_hdr_count = get_dim_size(&pb_hdr_dim); + if (pb_hdr_count < 0) { + mlog << Warning << "\n" << method_name + << "No extra header for PREPBUFR\n\n"; + return; } - else { - if (has_dim(f_in, nc_dim_nhdr_typ)) - obs_vars.hdr_typ_dim = get_nc_dim(f_in, nc_dim_nhdr_typ); // header dimension for message type - if (has_dim(f_in, nc_dim_nhdr_sid)) - obs_vars.hdr_sid_dim = get_nc_dim(f_in, nc_dim_nhdr_sid); // header dimension for station id - if (has_dim(f_in, nc_dim_nhdr_vld)) - obs_vars.hdr_vld_dim = get_nc_dim(f_in, nc_dim_nhdr_vld); // header dimension for valid time - if (has_dim(f_in, nc_dim_npbhdr)) - obs_vars.pb_hdr_dim = get_nc_dim(f_in, nc_dim_npbhdr); // header dimension for PB headers - } - obs_vars.obs_dim = get_nc_dim(f_in, nc_dim_nobs); // Observation array length - obs_vars.hdr_dim = get_nc_dim(f_in, nc_dim_nhdr); // Header array length - // Get netCDF header variables - obs_vars.hdr_typ_var = get_var(f_in, nc_var_hdr_typ); // Message type (String or int) - obs_vars.hdr_sid_var = get_var(f_in, nc_var_hdr_sid); // Station ID (String or int) - obs_vars.hdr_vld_var = get_var(f_in, nc_var_hdr_vld); // Valid time (String or int) - - ncVar = get_var(f_in, nc_var_hdr_lat); - if (IS_INVALID_NC(ncVar)) { - obs_vars.hdr_arr_var = get_var(f_in, nc_var_hdr_arr); // Header array - } else { - obs_vars.hdr_lat_var = ncVar; // Header array - obs_vars.hdr_lon_var = get_var(f_in, nc_var_hdr_lon); // Header array - obs_vars.hdr_elv_var = get_var(f_in, nc_var_hdr_elv); // Header array - obs_vars.hdr_typ_tbl_var = get_var(f_in, nc_var_hdr_typ_tbl); // Message type (String) - obs_vars.hdr_sid_tbl_var = get_var(f_in, nc_var_hdr_sid_tbl); // Station ID (String) - obs_vars.hdr_vld_tbl_var = get_var(f_in, nc_var_hdr_vld_tbl); // Valid time (String) - } - - // Get netCDF variables - ncVar = get_var(f_in, nc_var_obs_hid); - if (IS_INVALID_NC(ncVar)) { - obs_vars.obs_arr_var = get_var(f_in, nc_var_obs_arr); - } else { - obs_vars.obs_hid_var = ncVar; // Obs. header id array - ncVar = get_var(f_in, nc_var_obs_gc); - if (!IS_INVALID_NC(ncVar)) obs_vars.obs_gc_var = ncVar; // Obs. grib code array - ncVar = get_var(f_in, nc_var_obs_vid); - if (!IS_INVALID_NC(ncVar)) obs_vars.obs_vid_var = ncVar; // Obs. variable id array - obs_vars.obs_lvl_var = get_var(f_in, nc_var_obs_lvl); // Obs. pressure level array - obs_vars.obs_hgt_var = get_var(f_in, nc_var_obs_hgt); // Obs. highth array - obs_vars.obs_val_var = get_var(f_in, nc_var_obs_val); // Obs. value array - } - ncVar = get_var(f_in, nc_var_obs_qty); - if (!IS_INVALID_NC(ncVar)) obs_vars.obs_qty_var = ncVar; - ncVar = get_var(f_in, nc_var_obs_qty_tbl); - if (!IS_INVALID_NC(ncVar)) obs_vars.obs_qty_tbl_var = ncVar; + long offsets[1] = { 0 }; + long lengths[1] = { 1 }; + bool has_hdr_prpt_typ_var = !IS_INVALID_NC(hdr_prpt_typ_var); + bool has_hdr_irpt_typ_var = !IS_INVALID_NC(hdr_irpt_typ_var); + bool has_hdr_inst_typ_var = !IS_INVALID_NC(hdr_inst_typ_var); - // PrepBufr only headers - ncVar = get_var(f_in, nc_var_hdr_prpt_typ); - if (!IS_INVALID_NC(ncVar)) obs_vars.hdr_prpt_typ_var = ncVar; - ncVar = get_var(f_in, nc_var_hdr_irpt_typ); - if (!IS_INVALID_NC(ncVar)) obs_vars.hdr_irpt_typ_var = ncVar; - ncVar = get_var(f_in, nc_var_hdr_inst_typ); - if (!IS_INVALID_NC(ncVar)) obs_vars.hdr_inst_typ_var = ncVar; - - bool use_var_id = false; - if (!get_global_att(f_in, nc_att_use_var_id, use_var_id)) use_var_id = false; + if (has_hdr_prpt_typ_var) hdr_data.prpt_typ_array.extend(pb_hdr_count); + if (has_hdr_irpt_typ_var) hdr_data.irpt_typ_array.extend(pb_hdr_count); + if (has_hdr_inst_typ_var) hdr_data.inst_typ_array.extend(pb_hdr_count); - obs_vars.use_var_id = use_var_id; - nc_data_buffer.obs_vars = obs_vars; -} + // Read PB report type + int buf_size = ((pb_hdr_count > NC_BUFFER_SIZE_32K) + ? NC_BUFFER_SIZE_32K : (pb_hdr_count)); + int *hdr_prpt_typ_block = new int[buf_size]; + int *hdr_irpt_typ_block = new int[buf_size]; + int *hdr_inst_typ_block = new int[buf_size]; + for(int i_start=0; i_start NC_BUFFER_SIZE_32K) buf_size = NC_BUFFER_SIZE_32K; + offsets[0] = i_start; + lengths[0] = buf_size; -/////////////////////////////////////////////////////////////////////////////// + if (has_hdr_prpt_typ_var) { + // Get the corresponding header PB message type (string) + if(!get_nc_data(&hdr_prpt_typ_var, + hdr_prpt_typ_block, lengths, offsets)) { + mlog << Error << "\n" << method_name + << "trouble getting hdr_prpt_typ\n\n"; + exit(1); + } + } -void reset_header_buffer(int buf_size, bool reset_all) { - for (int i=0; i 0) + obs_cnt = 0; // obs. array length (fixed dimension if obs_cnt > 0) + //hdr_str_len = 0; // string length for header (message) type header } /////////////////////////////////////////////////////////////////////////////// -void write_header_to_nc(const NetcdfObsVars &obs_vars, - NcDataBuffer &data_buf, const int buf_size, const bool is_pb) +void NetcdfObsVars::write_header_to_nc(NcDataBuffer &data_buf, + const int buf_size, const bool is_pb) { + int dim_size; + bool is_prepbufr = is_pb || attr_pb2nc; long offsets[2] = { data_buf.hdr_data_offset, 0 }; long lengths[1] = { buf_size } ; - const string method_name = " write_header_to_nc()"; + const string method_name = " NetcdfObsVars::write_header_to_nc() "; + + ConcatString log_message; + if (is_prepbufr) { + log_message.add(", pb_hdr_data_offset: "); + log_message.add(data_buf.pb_hdr_data_offset); + log_message.add(", raw_hdr_cnt: "); + log_message.add(raw_hdr_cnt); + } + + mlog << Debug(7) << method_name << " buf_size: " << buf_size + << " is_prepbufr: " << is_prepbufr << ", hdr_data_offset: " + << data_buf.hdr_data_offset << log_message << "\n"; - mlog << Debug(7) << method_name << " buf_size: " << buf_size << "\n"; - //lengths[1] = HEADER_STR_LEN2; - if(!put_nc_data((NcVar *)&obs_vars.hdr_typ_var, (int *)data_buf.hdr_typ_buf, lengths, offsets)) { + dim_size = get_dim_size(&hdr_typ_var, 0); + if ((0 < dim_size) &&(dim_size < lengths[0])) { + mlog << Error << "\n" << method_name << " mismatching dimensions: allocated=" + << dim_size << ", data size=" << lengths[0] << " (hdr_typ)\n\n"; + exit(1); + } + else if(!put_nc_data((NcVar *)&hdr_typ_var, (int *)data_buf.hdr_typ_buf, lengths, offsets)) { mlog << Error << err_msg_message_type; exit(1); } - + // Station ID - if(!put_nc_data((NcVar *)&obs_vars.hdr_sid_var, (int *)data_buf.hdr_sid_buf, lengths, offsets)) { + if(!put_nc_data((NcVar *)&hdr_sid_var, (int *)data_buf.hdr_sid_buf, lengths, offsets)) { mlog << Error << err_msg_station_id; exit(1); } - + // Valid Time - if(!put_nc_data((NcVar *)&obs_vars.hdr_vld_var, (int *)data_buf.hdr_vld_buf, lengths, offsets)) { + if(!put_nc_data((NcVar *)&hdr_vld_var, (int *)data_buf.hdr_vld_buf, lengths, offsets)) { mlog << Error << err_msg_valid_time; exit(1); } - - // Write the header array which consists of the following: - // LAT LON ELV - - if(!put_nc_data((NcVar *)&obs_vars.hdr_lat_var, (float *)data_buf.hdr_lat_buf, lengths, offsets)) { + + // Write the header array which consists of the following: LAT LON ELV + dim_size = get_dim_size(&hdr_lat_var, 0); + if ((0 < dim_size) &&(dim_size < lengths[0])) { + mlog << Error << "\n" << method_name << " mismatching dimensions: allocated=" + << dim_size << ", data size=" << lengths[0] << " (hdr_lat)\n\n"; + exit(1); + } + else if(!put_nc_data((NcVar *)&hdr_lat_var, (float *)data_buf.hdr_lat_buf, lengths, offsets)) { mlog << Error << err_msg_hdr_arr; exit(1); } - if(!put_nc_data((NcVar *)&obs_vars.hdr_lon_var, (float *)data_buf.hdr_lon_buf, lengths, offsets)) { + if(!put_nc_data((NcVar *)&hdr_lon_var, (float *)data_buf.hdr_lon_buf, lengths, offsets)) { mlog << Error << err_msg_hdr_arr; exit(1); } - if(!put_nc_data((NcVar *)&obs_vars.hdr_elv_var, (float *)data_buf.hdr_elv_buf, lengths, offsets)) { + if(!put_nc_data((NcVar *)&hdr_elv_var, (float *)data_buf.hdr_elv_buf, lengths, offsets)) { mlog << Error << err_msg_hdr_arr; exit(1); } - - //for(int hi=0; hi buf_size) pb_hdr_len = buf_size; - - lengths[0] = pb_hdr_len; - if(!put_nc_data((NcVar *)&obs_vars.hdr_prpt_typ_var, data_buf.hdr_prpt_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the pb message type to the netCDF file\n\n"; - exit(1); + + if (is_prepbufr) { + if (0 == raw_hdr_cnt) { + mlog << Debug(6) << method_name + << "No header for PREPBUFR report/instrument\n"; } - if(!put_nc_data((NcVar *)&obs_vars.hdr_irpt_typ_var, data_buf.hdr_irpt_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the in message type to the netCDF file\n\n"; - exit(1); + else if (data_buf.hdr_data_offset == data_buf.pb_hdr_data_offset) { + int save_len = lengths[0]; + int pb_hdr_len = raw_hdr_cnt - offsets[0]; + if (pb_hdr_len > buf_size) pb_hdr_len = buf_size; + + lengths[0] = pb_hdr_len; + if(IS_VALID_NC(hdr_prpt_typ_var) && !put_nc_data((NcVar *)&hdr_prpt_typ_var, + data_buf.hdr_prpt_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the pb message type to the netCDF file\n\n"; + exit(1); + } + if(IS_VALID_NC(hdr_irpt_typ_var) && !put_nc_data((NcVar *)&hdr_irpt_typ_var, + data_buf.hdr_irpt_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the in message type to the netCDF file\n\n"; + exit(1); + } + if(IS_VALID_NC(hdr_inst_typ_var) && !put_nc_data((NcVar *)&hdr_inst_typ_var, + data_buf.hdr_inst_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the instrument type to the netCDF file\n\n"; + exit(1); + } + lengths[0] = save_len; + data_buf.pb_hdr_data_offset += pb_hdr_len; } - if(!put_nc_data((NcVar *)&obs_vars.hdr_inst_typ_var, data_buf.hdr_inst_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the instrument type to the netCDF file\n\n"; - exit(1); + else { + mlog << Debug(6) << method_name + << "Does not match header offsets for report/instrument: " << data_buf.hdr_data_offset + << " : " << data_buf.pb_hdr_data_offset << "\n"; } - lengths[0] = save_len; - data_buf.pb_hdr_data_offset += pb_hdr_len; } - + data_buf.hdr_data_offset += buf_size; data_buf.hdr_data_idx = 0; } /////////////////////////////////////////////////////////////////////////////// -// Saves the headers at NcHeaderData hdr_data -// -void write_nc_arr_headers(const NetcdfObsVars &obs_vars) +void NetcdfObsVars::write_table_vars (NcHeaderData &hdr_data, NcDataBuffer &data_buffer) { - int cur_hdr_idx = nc_data_buffer.cur_hdr_idx; - int buf_size = (cur_hdr_idx > OBS_BUFFER_SIZE) ? OBS_BUFFER_SIZE : cur_hdr_idx; - const string method_name = " write_nc_arr_headers()"; - - mlog << Debug(5) << method_name << " hdr_count: " << cur_hdr_idx - << ", typ_idx_array: " << hdr_data.typ_idx_array.n_elements() - << ", sid_idx_array: " << hdr_data.sid_idx_array.n_elements() - << ", vld_idx_array: " << hdr_data.vld_idx_array.n_elements() - << ", lat_array: " << hdr_data.lat_array.n_elements() - << ", lon_array: " << hdr_data.lon_array.n_elements() - << ", elv_array: " << hdr_data.elv_array.n_elements() - << "\n"; - - int hdr_data_idx = 0; - bool is_pb_hdr = (0 < hdr_data.prpt_typ_array.n_elements()) - && !IS_INVALID_NC(obs_vars.hdr_prpt_typ_var); - nc_data_buffer.obs_vars = obs_vars; - nc_data_buffer.hdr_buf_size = buf_size; - nc_data_buffer.hdr_data_idx = hdr_data_idx; - for (int index=0; index= buf_size) { - write_header_to_nc(obs_vars, nc_data_buffer, hdr_data_idx, is_pb_hdr); - hdr_data_idx = nc_data_buffer.hdr_data_idx; - } - } - - write_nc_buf_headers(obs_vars); + mlog << Debug(5) << "write_table_vars() is called. valid hdr_typ_tbl_var: " + << !IS_INVALID_NC(hdr_typ_tbl_var) << "\n"; + if (IS_INVALID_NC(hdr_typ_tbl_var)) + mlog << Warning << "\nwrite_table_vars() is called without creating variables\n\n"; + if (!IS_INVALID_NC(hdr_typ_tbl_var)) + write_nc_string_array (&hdr_typ_tbl_var, hdr_data.typ_array, HEADER_STR_LEN2); + if (!IS_INVALID_NC(hdr_sid_tbl_var)) + write_nc_string_array (&hdr_sid_tbl_var, hdr_data.sid_array, HEADER_STR_LEN2); + if (!IS_INVALID_NC(hdr_vld_tbl_var)) + write_nc_string_array (&hdr_vld_tbl_var, hdr_data.vld_array, HEADER_STR_LEN); + if (!IS_INVALID_NC(obs_qty_tbl_var)) + write_nc_string_array (&obs_qty_tbl_var, data_buffer.qty_data_array, HEADER_STR_LEN); } /////////////////////////////////////////////////////////////////////////////// -void write_nc_buf_headers (const NetcdfObsVars &obs_vars) +void NetcdfObsVars::write_obs_buffer(NcDataBuffer &data_buffer, const int buf_size) { - if (0 < nc_data_buffer.hdr_data_idx) { - write_header_to_nc(obs_vars, nc_data_buffer, nc_data_buffer.hdr_data_idx); - } - write_nc_table_vars((NetcdfObsVars &)obs_vars); -} - -/////////////////////////////////////////////////////////////////////////////// + long offsets[2] = { data_buffer.obs_data_offset, 0 }; + long lengths[1] = { buf_size} ; + const string method_name = " write_obs_buffer()"; -void write_nc_header (const NetcdfObsVars &obs_vars, - const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv) -{ - // Can't filter duplicated one because header index was - // assigned before checking - int hdr_index; - bool new_vld = false; - int hdr_data_idx = nc_data_buffer.hdr_data_idx; - - // Message type - if (!hdr_data.typ_array.has(hdr_typ, hdr_index, false)) { - hdr_index = hdr_data.typ_array.n_elements(); - hdr_data.typ_array.add(hdr_typ); + mlog << Debug(7) << method_name << " offset: " + << offsets[0] << ", " << offsets[1] << " buf_size: " << buf_size << "\n"; + mlog << Debug(7) << " obs_qty_var: " << GET_NC_NAME(obs_qty_var) << "\n"; + + if(!put_nc_data((NcVar *)&obs_qty_var, (int*)data_buffer.qty_idx_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << " -> " + << "error writing the quality flag to the " + << "netCDF file\n\n"; + exit(1); } - nc_data_buffer.hdr_typ_buf[hdr_data_idx] = hdr_index; - - // Station ID - if (!hdr_data.sid_array.has(hdr_sid, hdr_index, false)) { - hdr_index = hdr_data.sid_array.n_elements(); - hdr_data.sid_array.add(hdr_sid); + if(!put_nc_data((NcVar *)&obs_hid_var, (int*)data_buffer.obs_hid_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << " -> " + << "error writing the observation header index array to the " + << "netCDF file\n\n"; + exit(1); } - nc_data_buffer.hdr_sid_buf[hdr_data_idx] = hdr_index; - - // Valid Time - if (hdr_data.min_vld_time == -1 || hdr_data.min_vld_time > hdr_vld) { - if (hdr_data.min_vld_time == -1) hdr_data.max_vld_time = hdr_vld; - hdr_data.min_vld_time = hdr_vld; - new_vld = true; + bool use_var_id = !IS_INVALID_NC(obs_vid_var); + bool result = use_var_id + ? put_nc_data((NcVar *)&obs_vid_var, (int*)data_buffer.obs_vid_buf, lengths, offsets) + : put_nc_data((NcVar *)&obs_gc_var, (int*)data_buffer.obs_vid_buf, lengths, offsets); + if(!result) { + mlog << Error << "\n" << method_name << " -> " + << "error writing the observation " + << (use_var_id ? "variable_index" : "grib_code") + << " array to the netCDF file\n\n"; + exit(1); } - else if (hdr_data.max_vld_time < hdr_vld) { - hdr_data.max_vld_time = hdr_vld; - new_vld = true; + if(!put_nc_data((NcVar *)&obs_lvl_var, (float*)data_buffer.obs_lvl_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << " -> " + << "error writing the observation level array to the " + << "netCDF file\n\n"; + exit(1); } - - if (new_vld || !hdr_data.vld_num_array.has(hdr_vld, hdr_index, false)) { - hdr_index = hdr_data.vld_array.n_elements(); - hdr_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); - hdr_data.vld_num_array.add(hdr_vld); + if(!put_nc_data((NcVar *)&obs_hgt_var, (float*)data_buffer.obs_hgt_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << " -> " + << "error writing the observation hight array to the " + << "netCDF file\n\n"; + exit(1); } - nc_data_buffer.hdr_vld_buf[hdr_data_idx] = hdr_index; - - // Write the header array which consists of the following: - // LAT LON ELV - nc_data_buffer.hdr_lat_buf[hdr_data_idx] = (float) hdr_lat; - nc_data_buffer.hdr_lon_buf[hdr_data_idx] = (float) hdr_lon; - nc_data_buffer.hdr_elv_buf[hdr_data_idx] = (float) hdr_elv; - - hdr_data_idx++; - nc_data_buffer.hdr_data_idx = hdr_data_idx; - nc_data_buffer.cur_hdr_idx++; - - if (hdr_data_idx >= OBS_BUFFER_SIZE) { - write_header_to_nc(nc_data_buffer.obs_vars, nc_data_buffer, OBS_BUFFER_SIZE); + if(!put_nc_data((NcVar *)&obs_val_var, (float*)data_buffer.obs_val_buf, lengths, offsets)) { + mlog << Error << "\n" << method_name << " -> " + << "error writing the observation data array to the " + << "netCDF file\n\n"; + exit(1); } + + data_buffer.obs_data_offset += buf_size; + data_buffer.obs_data_idx = 0; +} + +//////////////////////////////////////////////////////////////////////// + +void NetcdfObsVars::write_obs_var_names(StringArray &obs_names) { + write_nc_string_array (&obs_var, obs_names, HEADER_STR_LEN2); +} + +//////////////////////////////////////////////////////////////////////// + +void NetcdfObsVars::write_obs_var_units(StringArray &units) { + write_nc_string_array (&unit_var, units, HEADER_STR_LEN2); +} + +//////////////////////////////////////////////////////////////////////// + +void NetcdfObsVars::write_obs_var_descriptions(StringArray &descriptions) { + write_nc_string_array (&desc_var, descriptions, HEADER_STR_LEN3); } - + +//////////////////////////////////////////////////////////////////////// +// End of NetcdfObsVars + /////////////////////////////////////////////////////////////////////////////// -void write_nc_observation(const NetcdfObsVars &obs_vars) +// struct NcHeaderData + +NcHeaderData::NcHeaderData() { - if (0 < nc_data_buffer.obs_data_idx){ - nc_data_buffer.obs_vars = obs_vars; - write_nc_obs_buffer(nc_data_buffer.obs_data_idx); - } + reset_counters(); } /////////////////////////////////////////////////////////////////////////////// -void write_nc_observation(const NetcdfObsVars &obs_vars, - const float obs_arr[OBS_ARRAY_LEN], const char *obs_qty) -{ - int qty_index; - int obs_data_idx = nc_data_buffer.obs_data_idx; - if (!nc_data_buffer.qty_data_array.has(obs_qty, qty_index, false)) { - qty_index = nc_data_buffer.qty_data_array.n_elements(); - nc_data_buffer.qty_data_array.add(obs_qty); - } - nc_data_buffer.qty_idx_buf[obs_data_idx] = qty_index; - - //for (int idx=0; idx= OBS_BUFFER_SIZE) { - write_nc_obs_buffer(OBS_BUFFER_SIZE); - } +void NcHeaderData::clear() { + reset_counters(); + + typ_array.clear(); + sid_array.clear(); + vld_array.clear(); + vld_num_array.clear(); + typ_idx_array.clear(); + sid_idx_array.clear(); + vld_idx_array.clear(); + lat_array.clear(); + lon_array.clear(); + elv_array.clear(); + prpt_typ_array.clear(); + irpt_typ_array.clear(); + inst_typ_array.clear(); } /////////////////////////////////////////////////////////////////////////////// -void write_nc_table_vars (NetcdfObsVars &obs_vars) +void NcHeaderData::reset_counters() { + valid_point_obs = false; + typ_len = 0; + sid_len = 0; + vld_len = 0; + strl_len = 0; + strll_len = 0; + hdr_count = 0; + + min_vld_time = -1; + max_vld_time = -1; +} + + +/////////////////////////////////////////////////////////////////////////////// + +long count_nc_headers(vector< Observation > &observations) { - mlog << Debug(5) << " write_nc_table_vars() is called. valid hdr_typ_tbl_var: " - << !IS_INVALID_NC(obs_vars.hdr_typ_tbl_var) << "\n"; - if (IS_INVALID_NC(obs_vars.hdr_typ_tbl_var)) - mlog << Warning << "\nwrite_nc_table_vars() is called without creating variables\n\n"; - if (!IS_INVALID_NC(obs_vars.hdr_typ_tbl_var)) - write_nc_string_array (&obs_vars.hdr_typ_tbl_var, hdr_data.typ_array, HEADER_STR_LEN2); - if (!IS_INVALID_NC(obs_vars.hdr_sid_tbl_var)) - write_nc_string_array (&obs_vars.hdr_sid_tbl_var, hdr_data.sid_array, HEADER_STR_LEN2); - if (!IS_INVALID_NC(obs_vars.hdr_vld_tbl_var)) - write_nc_string_array (&obs_vars.hdr_vld_tbl_var, hdr_data.vld_array, HEADER_STR_LEN); - if (!IS_INVALID_NC(obs_vars.obs_qty_tbl_var)) - write_nc_string_array (&obs_vars.obs_qty_tbl_var, nc_data_buffer.qty_data_array, HEADER_STR_LEN); + long nhdr = 0; + + string prev_header_type = ""; + string prev_station_id = ""; + time_t prev_valid_time = 0; + double prev_latitude = bad_data_double; + double prev_longitude = bad_data_double; + double prev_elevation = bad_data_double; + const string method_name = " count_nc_headers()"; + + for (vector< Observation >::iterator obs = observations.begin(); + obs != observations.end(); ++obs) + { + if (obs->getHeaderType() != prev_header_type || + obs->getStationId() != prev_station_id || + obs->getValidTime() != prev_valid_time || + !is_eq(obs->getLatitude(), prev_latitude) || + !is_eq(obs->getLongitude(), prev_longitude) || + !is_eq(obs->getElevation(), prev_elevation)) + { + nhdr++; + + prev_header_type = obs->getHeaderType(); + prev_station_id = obs->getStationId(); + prev_valid_time = obs->getValidTime(); + prev_latitude = obs->getLatitude(); + prev_longitude = obs->getLongitude(); + prev_elevation = obs->getElevation(); + } + obs->setHeaderIndex(nhdr-1); + } /* endfor - obs */ + + return nhdr; +} + + +/////////////////////////////////////////////////////////////////////////////// + +bool is_using_var_id(const char * nc_name) { + bool use_var_id = false; + if (!get_global_att(nc_name, nc_att_use_var_id, use_var_id)) { + use_var_id = false; + } + return use_var_id; +} + +//////////////////////////////////////////////////////////////////////// + +bool is_using_var_id(NcFile *nc_file) { + bool use_var_id = false; + if (!get_global_att(nc_file, nc_att_use_var_id, use_var_id)) { + use_var_id = false; + } + return use_var_id; } /////////////////////////////////////////////////////////////////////////////// int write_nc_string_array (NcVar *ncVar, StringArray &strArray, const int str_len) { - //float obs_arr[obs_arr_len]; const string method_name = " write_nc_string_array() "; int data_count = strArray.n_elements(); int max_buf_size = (1024 * 8); @@ -1278,7 +1380,7 @@ int write_nc_string_array (NcVar *ncVar, StringArray &strArray, const int str_le mlog << Debug(7) << method_name << " " << GET_NC_NAME_P(ncVar) << " data count: " << data_count << "\n"; - + // Initialize data_buf for (int indexX=0; indexX 0) { lengths[0] = (data_count <= max_buf_size) ? data_count : (data_count % buf_size); mlog << Debug(7) << method_name << " Save to NetCDF. offsets: " << offsets[0] @@ -1330,169 +1432,4 @@ int write_nc_string_array (NcVar *ncVar, StringArray &strArray, const int str_le return processed_count; } -/////////////////////////////////////////////////////////////////////////////// - -void write_nc_obs_buffer(const int buf_size) -{ - const NetcdfObsVars &obs_vars = nc_data_buffer.obs_vars; - long offsets[2] = { nc_data_buffer.obs_data_offset, 0 }; - long lengths[1] = { buf_size} ; - const string method_name = " write_nc_obs_buffer()"; - - mlog << Debug(7) << method_name << " offset: " - << offsets[0] << ", " << offsets[1] << " buf_size: " << buf_size << "\n"; - mlog << Debug(7) << " obs_qty_var: " << GET_NC_NAME(obs_vars.obs_qty_var) << "\n"; - - //lengths[1] = HEADER_STR_LEN; - if(!put_nc_data((NcVar *)&obs_vars.obs_qty_var, (int*)nc_data_buffer.qty_idx_buf, lengths, offsets)) { - mlog << Error << "\n" << method_name << " -> " - << "error writing the quality flag to the " - << "netCDF file\n\n"; - exit(1); - } - //lengths[1] = OBS_ARRAY_LEN; - if(!put_nc_data((NcVar *)&obs_vars.obs_hid_var, (int*)nc_data_buffer.obs_hid_buf, lengths, offsets)) { - mlog << Error << "\n" << method_name << " -> " - << "error writing the observation header index array to the " - << "netCDF file\n\n"; - exit(1); - } - bool use_var_id = !IS_INVALID_NC(obs_vars.obs_vid_var); - bool result = use_var_id - ? put_nc_data((NcVar *)&obs_vars.obs_vid_var, (int*)nc_data_buffer.obs_vid_buf, lengths, offsets) - : put_nc_data((NcVar *)&obs_vars.obs_gc_var, (int*)nc_data_buffer.obs_vid_buf, lengths, offsets); - if(!result) { - mlog << Error << "\n" << method_name << " -> " - << "error writing the observation " - << (use_var_id ? "variable_index" : "grib_code") - << " array to the netCDF file\n\n"; - exit(1); - } - if(!put_nc_data((NcVar *)&obs_vars.obs_lvl_var, (float*)nc_data_buffer.obs_lvl_buf, lengths, offsets)) { - mlog << Error << "\n" << method_name << " -> " - << "error writing the observation level array to the " - << "netCDF file\n\n"; - exit(1); - } - if(!put_nc_data((NcVar *)&obs_vars.obs_hgt_var, (float*)nc_data_buffer.obs_hgt_buf, lengths, offsets)) { - mlog << Error << "\n" << method_name << " -> " - << "error writing the observation hight array to the " - << "netCDF file\n\n"; - exit(1); - } - if(!put_nc_data((NcVar *)&obs_vars.obs_val_var, (float*)nc_data_buffer.obs_val_buf, lengths, offsets)) { - mlog << Error << "\n" << method_name << " -> " - << "error writing the observation data array to the " - << "netCDF file\n\n"; - exit(1); - } - - nc_data_buffer.obs_data_offset += buf_size; - nc_data_buffer.obs_data_idx = 0; -} - -/////////////////////////////////////////////////////////////////////////////// - -int write_nc_observations(const NetcdfObsVars &obs_vars, - const vector< Observation > observations, - const bool use_var_id, const bool do_header) -{ - int prev_hdr_idx = -1; - string prev_header_type = ""; - string prev_station_id = ""; - ConcatString obs_qty; - int headerOffset = nc_data_buffer.cur_hdr_idx; - //float obs_arr[obs_arr_len]; - const string method_name = " write_nc_observations()"; - - int obs_buf_size = observations.size(); - mlog << Debug(7) << method_name << " obs_count: " << obs_buf_size << "\n"; - if (obs_buf_size > OBS_BUFFER_SIZE) obs_buf_size = OBS_BUFFER_SIZE; - - //if (reset) { - // nc_data_buffer.obs_vars = obs_vars; - // nc_data_buffer.obs_buf_size = obs_buf_size; - // nc_data_buffer.obs_data_idx = 0; - // nc_data_buffer.obs_data_offset = 0; - // nc_data_buffer.hdr_data_idx = 0; - // nc_data_buffer.hdr_data_offset = 0; - // - // nc_data_buffer.processed_count =0; - //} - float obs_arr[OBS_ARRAY_LEN]; - bool header_to_vector = IS_INVALID_NC(obs_vars.hdr_arr_var) - || IS_INVALID_NC(obs_vars.hdr_lat_var); - mlog << Debug(5) << method_name << " do_header: " << do_header - << " header_to_vector: " << header_to_vector << "\n"; - for (vector< Observation >::const_iterator obs = observations.begin(); - obs != observations.end(); ++obs) - { - nc_data_buffer.processed_count++; - - if (do_header) { - if (obs->getHeaderIndex() != prev_hdr_idx) { - mlog << Debug(9) << method_name << " obs->getHeaderIndex(): " - << obs->getHeaderIndex() << " at obs " << nc_data_buffer.processed_count << "\n"; - prev_hdr_idx = obs->getHeaderIndex(); - if (header_to_vector) { - add_nc_header_to_array( - obs->getHeaderType().c_str(), - obs->getStationId().c_str(), - obs->getValidTime(), - obs->getLatitude(), - obs->getLongitude(), - obs->getElevation()); - } - else { - write_nc_header( - obs_vars, - obs->getHeaderType().c_str(), - obs->getStationId().c_str(), - obs->getValidTime(), - obs->getLatitude(), - obs->getLongitude(), - obs->getElevation()); - } - } - } - - obs_arr[0] = obs->getHeaderIndex(); - obs_arr[1] = (use_var_id ? obs->getVarCode() : obs->getGribCode()); - obs_arr[2] = obs->getPressureLevel(); - obs_arr[3] = obs->getHeight(); - obs_arr[4] = obs->getValue(); - obs_qty = (obs->getQualityFlag().length() == 0 ? na_str : obs->getQualityFlag().c_str()); - if (do_header) obs_arr[0] += headerOffset; - - write_nc_observation(obs_vars, obs_arr, obs_qty.text()); - - } /* endfor - obs */ - - if (nc_data_buffer.obs_data_idx > 0) { - write_nc_obs_buffer(nc_data_buffer.obs_data_idx); - } - - //Caller handles writing headers - - return nc_data_buffer.processed_count; -} - -//////////////////////////////////////////////////////////////////////// - -void write_obs_var_names(NetcdfObsVars &obs_vars, StringArray &obs_names) { - write_nc_string_array (&obs_vars.obs_var, obs_names, HEADER_STR_LEN2); -} - -//////////////////////////////////////////////////////////////////////// - -void write_obs_var_units(NetcdfObsVars &obs_vars, StringArray &units) { - write_nc_string_array (&obs_vars.unit_var, units, HEADER_STR_LEN2); -} - -//////////////////////////////////////////////////////////////////////// - -void write_obs_var_descriptions(NetcdfObsVars &obs_vars, StringArray &descriptions) { - write_nc_string_array (&obs_vars.desc_var, descriptions, HEADER_STR_LEN3); -} - //////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.h b/met/src/libcode/vx_nc_obs/nc_obs_util.h index 58c5748e24..4985639c09 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.h +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.h @@ -16,9 +16,101 @@ #include using namespace netCDF; +#include "nc_summary.h" + +//////////////////////////////////////////////////////////////////////// + + +static const char empty_name[] = ""; + + //////////////////////////////////////////////////////////////////////// // struct definition +struct NcDataBuffer { + int processed_count; + int obs_count; + int obs_buf_size; + int cur_obs_idx; + int obs_data_idx; + int obs_data_offset; + int hdr_count; + int hdr_buf_size; + int cur_hdr_idx; + int hdr_data_idx; + int hdr_data_offset; + int pb_hdr_count; + int pb_hdr_data_offset; + + int prev_hdr_vld; + char prev_hdr_typ_buf[HEADER_STR_LEN2]; + char prev_hdr_sid_buf[HEADER_STR_LEN2]; + char prev_hdr_vld_buf[HEADER_STR_LEN]; + float prev_hdr_arr_buf[HDR_ARRAY_LEN]; + + char hdr_typ_str_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN2]; + char hdr_sid_str_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN2]; + char hdr_vld_str_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN]; + float hdr_arr_buf[OBS_BUFFER_SIZE][HDR_ARRAY_LEN]; + float obs_data_buf[OBS_BUFFER_SIZE][OBS_ARRAY_LEN]; + char qty_data_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN]; + + StringArray qty_data_array; + int hdr_typ_buf[OBS_BUFFER_SIZE]; + int hdr_sid_buf[OBS_BUFFER_SIZE]; + int hdr_vld_buf[OBS_BUFFER_SIZE]; + float hdr_lat_buf[OBS_BUFFER_SIZE]; + float hdr_lon_buf[OBS_BUFFER_SIZE]; + float hdr_elv_buf[OBS_BUFFER_SIZE]; + int hdr_prpt_typ_buf[OBS_BUFFER_SIZE]; + int hdr_irpt_typ_buf[OBS_BUFFER_SIZE]; + int hdr_inst_typ_buf[OBS_BUFFER_SIZE]; + int obs_hid_buf[OBS_BUFFER_SIZE]; + int obs_vid_buf[OBS_BUFFER_SIZE]; + int qty_idx_buf[OBS_BUFFER_SIZE]; + float obs_lvl_buf[OBS_BUFFER_SIZE]; + float obs_hgt_buf[OBS_BUFFER_SIZE]; + float obs_val_buf[OBS_BUFFER_SIZE]; + + NcDataBuffer(); + void reset_counters(); +}; + +//////////////////////////////////////////////////////////////////////// + +struct NcHeaderData { + bool valid_point_obs; + int typ_len; + int sid_len; + int vld_len; + int strl_len; + int strll_len; + int min_vld_time; + int max_vld_time; + int hdr_count; + int hdr_type_count; + + StringArray typ_array; + StringArray sid_array; + StringArray vld_array; + IntArray vld_num_array; + IntArray typ_idx_array; + IntArray sid_idx_array; + IntArray vld_idx_array; + NumArray lat_array; + NumArray lon_array; + NumArray elv_array; + IntArray prpt_typ_array; + IntArray irpt_typ_array; + IntArray inst_typ_array; + + NcHeaderData(); + void clear(); + void reset_counters(); +}; + +//////////////////////////////////////////////////////////////////////// + struct NetcdfObsVars { bool attr_agl ; bool attr_pb2nc ; @@ -26,7 +118,8 @@ struct NetcdfObsVars { int hdr_cnt ; // header array count (fixed dimension if hdr_cnt > 0) int obs_cnt ; // obs. array count (fixed dimension if obs_cnt > 0) int raw_hdr_cnt ; // raw data (PrepBufr) header array count - + int deflate_level; + NcDim strl_dim ; // header string dimension (16 bytes) NcDim strl2_dim ; // header string dimension (40 bytes) NcDim strl3_dim ; // header string dimension (80 bytes) @@ -38,7 +131,7 @@ struct NetcdfObsVars { NcDim obs_dim ; // Observation array length (V1.0) NcDim hdr_dim ; // Header array length (V1.0) NcDim pb_hdr_dim ; // PrefBufr Header array length (V1.2) - + NcVar hdr_typ_tbl_var ; // Message type (string) (V1.1) NcVar hdr_sid_tbl_var ; // Station ID (string) (V1.1) NcVar hdr_vld_tbl_var ; // Valid time (string) (V1.1) @@ -65,155 +158,87 @@ struct NetcdfObsVars { NcVar obs_var ; // Observation variable name (V1.1) NcVar unit_var ; // The unit of the observation variable (V1.1) NcVar desc_var ; // The description of the observation variable (V1.1) -}; -struct NcHeaderData { - int typ_len; - int sid_len; - int vld_len; - int strl_len; - int strll_len; - int min_vld_time; - int max_vld_time; - - StringArray typ_array; - StringArray sid_array; - StringArray vld_array; - IntArray vld_num_array; - IntArray typ_idx_array; - IntArray sid_idx_array; - IntArray vld_idx_array; - NumArray lat_array; - NumArray lon_array; - NumArray elv_array; - IntArray prpt_typ_array; - IntArray irpt_typ_array; - IntArray inst_typ_array; -}; + NetcdfObsVars(); + bool is_valid(bool exit_on_error=false); + void reset(bool _use_var_id = true); -struct NcDataBuffer { - NetcdfObsVars obs_vars; - int processed_count; - int obs_count; - int obs_buf_size; - int cur_obs_idx; - int obs_data_idx; - int obs_data_offset; - int hdr_count; - int hdr_buf_size; - int cur_hdr_idx; - int hdr_data_idx; - int hdr_data_offset; - int pb_hdr_count; - int pb_hdr_data_offset; + void create_dimensions(NcFile *f_out); + void create_hdr_vars (NcFile *f_out, const int hdr_count); + void create_obs_vars (NcFile *f_out); + void create_obs_name_vars (NcFile *f_out, const int var_count, const int unit_count); + void create_table_vars (NcFile *f_out, NcHeaderData &hdr_data, NcDataBuffer &data_buffer); + void create_pb_hdrs (NcFile *f_out, const int hdr_count); + NcDim create_var_obs_var (NcFile *f_out, int var_count); - int prev_hdr_vld; - char prev_hdr_typ_buf[HEADER_STR_LEN2]; - char prev_hdr_sid_buf[HEADER_STR_LEN2]; - char prev_hdr_vld_buf[HEADER_STR_LEN]; - float prev_hdr_arr_buf[HDR_ARRAY_LEN]; + int get_hdr_index(); + int get_obs_index(); + + void read_dims_vars(NcFile *f_in); + void read_header_data(NcHeaderData &hdr_data); + bool read_obs_data(int buf_size, int offset, int qty_len, float *obs_arr, + int *qty_idx_arr, char *obs_qty_buf); + void read_pb_hdr_data(NcHeaderData &hdr_data); + + void write_header_to_nc(NcDataBuffer &data_buf, const int buf_size, const bool is_pb = false); + void write_obs_buffer(NcDataBuffer &data_buffer, const int buf_size); + void write_obs_var_names(StringArray &obs_names); + void write_obs_var_units(StringArray &units); + void write_obs_var_descriptions(StringArray &descriptions); + void write_table_vars(NcHeaderData &hdr_data, NcDataBuffer &data_buffer); + +}; // NetcdfObsVars + +//////////////////////////////////////////////////////////////////////// + +struct NcPointObsData { + int obs_cnt; + bool is_obs_array; - char hdr_typ_str_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN2]; - char hdr_sid_str_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN2]; - char hdr_vld_str_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN]; - float hdr_arr_buf[OBS_BUFFER_SIZE][HDR_ARRAY_LEN]; - float obs_data_buf[OBS_BUFFER_SIZE][OBS_ARRAY_LEN]; - char qty_data_buf[OBS_BUFFER_SIZE][HEADER_STR_LEN]; + int *obs_ids; // grib_code or var_id + int *obs_hids; + int *obs_qids; + float *obs_lvls; + float *obs_hgts; + float *obs_vals; + float *obs_arr; // nobs * 5 + StringArray var_names; + StringArray qty_names; - StringArray qty_data_array; - int hdr_typ_buf[OBS_BUFFER_SIZE]; - int hdr_sid_buf[OBS_BUFFER_SIZE]; - int hdr_vld_buf[OBS_BUFFER_SIZE]; - float hdr_lat_buf[OBS_BUFFER_SIZE]; - float hdr_lon_buf[OBS_BUFFER_SIZE]; - float hdr_elv_buf[OBS_BUFFER_SIZE]; - int hdr_prpt_typ_buf[OBS_BUFFER_SIZE]; - int hdr_irpt_typ_buf[OBS_BUFFER_SIZE]; - int hdr_inst_typ_buf[OBS_BUFFER_SIZE]; - int obs_hid_buf[OBS_BUFFER_SIZE]; - int obs_vid_buf[OBS_BUFFER_SIZE]; - int qty_idx_buf[OBS_BUFFER_SIZE]; - float obs_lvl_buf[OBS_BUFFER_SIZE]; - float obs_hgt_buf[OBS_BUFFER_SIZE]; - float obs_val_buf[OBS_BUFFER_SIZE]; + NcPointObsData(); + void clear(); + void clear_numbers(); + void clear_strings(); + float get_obs_val(int index); + bool read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop=true); + bool read_obs_data_table_lookups(NetcdfObsVars obs_vars, bool stop=true); }; //////////////////////////////////////////////////////////////////////// + // extern variables //////////////////////////////////////////////////////////////////////// -extern bool add_nc_header_to_array - (const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv); extern bool add_nc_header_prepbufr (const int pb_report_type, - const int in_report_type, const int instrument_type); -//extern bool is_same_header -// (const char *hdr_typ, const char *hdr_sid, const unixtime hdr_vld, -// const float hdr_lat, const float hdr_lon, const float hdr_elv); -//extern bool is_same_header -// (const char *hdr_typ, const char *hdr_sid, const char *hdr_vld, -// const float hdr_lat, const float hdr_lon, const float hdr_elv); + const int in_report_type, const int instrument_type); -extern int check_nc_dims_vars(const NetcdfObsVars obs_vars); - -extern void clear_header_data(NcHeaderData *); - extern long count_nc_headers (vector< Observation > &observations); -extern void create_nc_hdr_vars (NetcdfObsVars &, NcFile *, const int hdr_count, const int deflate_level=0); -extern void create_nc_name_vars(NetcdfObsVars &, NcFile *, const int var_count = 0, - const int unit_count=0, const int deflate_level=0); -extern void create_nc_obs_name_vars(NetcdfObsVars &, NcFile *, - const int var_count, const int unit_count, const int deflate_level=0); -extern NcDim create_nc_obs_var_var(NetcdfObsVars &, NcFile *, int var_count, const int deflate_level); -extern void create_nc_obs_vars (NetcdfObsVars &, NcFile *, const int deflate_level=0, const bool use_var_id=true); -extern void create_nc_table_vars(NetcdfObsVars &, NcFile *, const int deflate_level=0); - -extern void create_nc_pb_hdrs (NetcdfObsVars &, NcFile *, const int hdr_count, const int deflate_level=0); - -extern NcDataBuffer *get_nc_data_buffer(); -extern NcHeaderData *get_hdr_data_buffer(); - extern int get_nc_hdr_cur_index(); extern int get_nc_obs_buf_index(); -extern NcHeaderData get_nc_hdr_data(NetcdfObsVars obs_vars); -extern void get_nc_pb_hdr_data (NetcdfObsVars obs_vars, NcHeaderData *header_data); - -extern void init_nc_dims_vars_config(NetcdfObsVars &, bool use_var_id = true); extern bool is_using_var_id (const char * nc_name); - -extern void nc_obs_initialize (); - -extern bool read_nc_obs_data(NetcdfObsVars obs_vars, int buf_size, int offset, - int qty_len, float *obs_arr, int *qty_idx_arr, char *obs_qty_buf); - -extern void read_nc_dims_vars (NetcdfObsVars &, NcFile *); +extern bool is_using_var_id (NcFile * nc_file); extern void reset_header_buffer(int buf_size, bool reset_all=false); -extern void set_header_buffer(int buf_size, bool reset_all=false); +extern void set_header_buffer (int buf_size, bool reset_all=false); -extern void write_header_to_nc (const NetcdfObsVars &, NcDataBuffer &data_buf, - const int buf_size, const bool is_pb = false); +extern string seconds_to_time_string(const int secs); -extern void write_nc_table_vars (NetcdfObsVars &); -extern void write_nc_arr_headers (const NetcdfObsVars &); -extern void write_nc_buf_headers (const NetcdfObsVars &); -extern void write_nc_header (const NetcdfObsVars &, - const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv); extern void write_nc_obs_buffer (const int buf_size); -extern int write_nc_observations (const NetcdfObsVars &, const vector< Observation > observations, - const bool use_var_idx = true, const bool do_header = true); -extern void write_nc_observation (const NetcdfObsVars &, - const float obs_arr[OBS_ARRAY_LEN], const char *obs_qty); -extern void write_nc_observation (const NetcdfObsVars &); -extern int write_nc_string_array (NcVar *ncVar, StringArray &strArray, const int str_len); - -extern void write_obs_var_names(NetcdfObsVars &, StringArray &); -extern void write_obs_var_units(NetcdfObsVars &, StringArray &); -extern void write_obs_var_descriptions(NetcdfObsVars &, StringArray &); +extern int write_nc_string_array (NcVar *ncVar, StringArray &strArray, + const int str_len); #endif /* __NC_OBS_UTIL_H__ */ diff --git a/met/src/libcode/vx_nc_obs/nc_point_obs.cc b/met/src/libcode/vx_nc_obs/nc_point_obs.cc new file mode 100644 index 0000000000..b665e9d2e9 --- /dev/null +++ b/met/src/libcode/vx_nc_obs/nc_point_obs.cc @@ -0,0 +1,167 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include + +#include "vx_log.h" +#include "is_bad_data.h" + +#include "nc_point_obs.h" + +//////////////////////////////////////////////////////////////////////// + + + // + // Code for class MetNcPointObs + // + + +//////////////////////////////////////////////////////////////////////// + +MetNcPointObs::MetNcPointObs() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +MetNcPointObs::~MetNcPointObs() { + close(); +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObs::init_from_scratch() { + obs_nc = (NcFile *) 0; + + nobs = 0; + nhdr = 0; + qty_length = 0; + keep_nc = false; + use_var_id = false; + use_arr_vars = false; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObs::close() { + if ( !keep_nc && obs_nc ) { + delete obs_nc; + obs_nc = (NcFile *) 0; + } + + obs_data.clear(); + header_data.clear(); + return; +} + +//////////////////////////////////////////////////////////////////////// + +int MetNcPointObs::get_qty_length() { + qty_length = get_nc_string_length(&obs_vars.obs_qty_tbl_var); + return qty_length; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObs::get_header(int header_offset, float hdr_arr[HDR_ARRAY_LEN], + ConcatString &hdr_typ_str, ConcatString &hdr_sid_str, ConcatString &hdr_vld_str) { + int hdr_idx; + + // Read the corresponding header array for this observation + hdr_arr[0] = header_data.lat_array[header_offset]; + hdr_arr[1] = header_data.lon_array[header_offset]; + hdr_arr[2] = header_data.elv_array[header_offset]; + + // Read the corresponding header type for this observation + hdr_idx = use_arr_vars ? header_offset : header_data.typ_idx_array[header_offset]; + hdr_typ_str = header_data.typ_array[hdr_idx]; + + // Read the corresponding header Station ID for this observation + hdr_idx = use_arr_vars ? header_offset : header_data.sid_idx_array[header_offset]; + hdr_sid_str = header_data.sid_array[hdr_idx]; + + // Read the corresponding valid time for this observation + hdr_idx = use_arr_vars ? header_offset : header_data.vld_idx_array[header_offset]; + hdr_vld_str = header_data.vld_array[hdr_idx]; + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObs::get_header_type(int header_offset, int hdr_typ_arr[HDR_TYPE_ARR_LEN]) { + int hdr_idx; + // Read the header integer types + hdr_typ_arr[0] = (header_data.prpt_typ_array.n() > header_offset ? + header_data.prpt_typ_array[header_offset] : bad_data_int); + hdr_typ_arr[1] = (header_data.irpt_typ_array.n() > header_offset ? + header_data.irpt_typ_array[header_offset] : bad_data_int); + hdr_typ_arr[2] = (header_data.inst_typ_array.n() > header_offset ? + header_data.inst_typ_array[header_offset] : bad_data_int); + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObs::get_lats(float *hdr_lats) { + for (int idx=0; idx + +#include "observation.h" +#include "nc_utils.h" +#include "nc_obs_util.h" +#include "nc_var_info.h" +#include "vx_summary.h" + + +//////////////////////////////////////////////////////////////////////// + + +class MetNcPointObs { + + protected: + + int nobs; + int nhdr; + int qty_length; + bool keep_nc; + bool use_var_id; + bool use_arr_vars; + NcFile *obs_nc; // allocated + + NetcdfObsVars obs_vars; + NcPointObsData obs_data; + NcHeaderData header_data; + + void init_from_scratch(); + + public: + + MetNcPointObs(); + ~MetNcPointObs(); + + bool open(const char * filename); + void close(); + bool set_netcdf(NcFile *nc_file, bool _keep_nc=false); + + int get_buf_size(); + int get_hdr_cnt(); + int get_grib_code_or_var_index(const float obs_arr[OBS_ARRAY_LEN]); + NcHeaderData get_header_data(); + bool get_header(int header_offset, float hdr_arr[HDR_ARRAY_LEN], + ConcatString &hdr_typ_str, ConcatString &hdr_sid_str, + ConcatString &hdr_vld_str); + int get_header_offset(const float obs_arr[OBS_ARRAY_LEN]); + bool get_header_type(int header_offset, int hdr_typ_arr[HDR_TYPE_ARR_LEN]); + bool get_lats(float *hdr_lats); + bool get_lons(float *hdr_lons); + int get_obs_cnt(); + NcPointObsData get_point_obs_data(); + StringArray get_qty_data(); + int get_qty_length(); + StringArray get_var_names(); + + bool is_same_obs_values(const float obs_arr1[OBS_ARRAY_LEN], const float obs_arr2[OBS_ARRAY_LEN]); + bool is_using_var_id(); + bool is_using_obs_arr(); + + void set_grib_code_or_var_index(float obs_arr[OBS_ARRAY_LEN], int grib_code); + // variables + + // data + +}; // MetNcPointObs + +//////////////////////////////////////////////////////////////////////// + +inline NcHeaderData MetNcPointObs::get_header_data() { return header_data; } +inline int MetNcPointObs::get_buf_size() { return OBS_BUFFER_SIZE; } +inline int MetNcPointObs::get_grib_code_or_var_index(const float obs_arr[OBS_ARRAY_LEN]) { return obs_arr[1]; }; +inline int MetNcPointObs::get_hdr_cnt() { return nhdr; } +inline int MetNcPointObs::get_header_offset(const float obs_arr[OBS_ARRAY_LEN]) { return obs_arr[0]; }; +inline int MetNcPointObs::get_obs_cnt() { return nobs; } +inline NcPointObsData MetNcPointObs::get_point_obs_data() { return obs_data; } +inline StringArray MetNcPointObs::get_qty_data() { return obs_data.qty_names; } +inline StringArray MetNcPointObs::get_var_names() { return obs_data.var_names; } +inline bool MetNcPointObs::is_using_obs_arr() { return use_arr_vars; } +inline bool MetNcPointObs::is_using_var_id() { return use_var_id; } +inline void MetNcPointObs::set_grib_code_or_var_index(float obs_arr[OBS_ARRAY_LEN], int grib_code) { obs_arr[1] = grib_code; } + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __NC_POINT_OBS_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_nc_obs/nc_point_obs_in.cc b/met/src/libcode/vx_nc_obs/nc_point_obs_in.cc new file mode 100644 index 0000000000..3345edb5a7 --- /dev/null +++ b/met/src/libcode/vx_nc_obs/nc_point_obs_in.cc @@ -0,0 +1,113 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include + +#include "vx_log.h" + +#include "nc_point_obs_in.h" + +//////////////////////////////////////////////////////////////////////// + + + // + // Code for class MetNcPointObs + // + + +//////////////////////////////////////////////////////////////////////// + +MetNcPointObsIn::MetNcPointObsIn() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +MetNcPointObsIn::~MetNcPointObsIn() { + close(); +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsIn::check_nc(const char *nc_name, const char *caller) { + bool exit_on_error = false; + bool valid = obs_vars.is_valid(exit_on_error); + if (!valid) { + mlog << Error << "\n" << (0 != caller ? caller : "") << " -> " + << "missing core data from the netCDF file: " + << nc_name << "\n\n"; + exit(1); + } + return valid; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsIn::read_dim_headers() { + bool status = false; + if( IS_VALID_NC_P(obs_nc) ) { + status = true; + obs_vars.read_dims_vars(obs_nc); + nobs = obs_vars.obs_cnt = GET_NC_SIZE(obs_vars.obs_dim); + nhdr = obs_vars.hdr_cnt = GET_NC_SIZE(obs_vars.hdr_dim); + obs_vars.use_var_id = use_var_id = IS_VALID_NC(obs_vars.obs_vid_var); + use_arr_vars = IS_VALID_NC(obs_vars.obs_arr_var); + obs_vars.read_header_data(header_data); + } + return status; +} + +////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsIn::read_obs_data() { + bool status = read_obs_data_numbers() && read_obs_data_table_lookups(); + return status; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsIn::read_obs_data(int buf_size, int start, float *obs_arr_block, + int *obs_qty_idx_block, char *obs_qty_str_block) { + return obs_vars.read_obs_data(buf_size, start, qty_length, obs_arr_block, + obs_qty_idx_block, obs_qty_str_block); +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsIn::read_obs_data_numbers() { + bool status = false; + if( IS_VALID_NC_P(obs_nc) ) { + status = obs_data.read_obs_data_numbers(obs_vars); + } + return status; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsIn::read_obs_data_table_lookups() { + bool status = false; + if( IS_VALID_NC_P(obs_nc) ) { + status = obs_data.read_obs_data_table_lookups(obs_vars); + } + return status; +} + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_nc_obs/nc_point_obs_in.h b/met/src/libcode/vx_nc_obs/nc_point_obs_in.h new file mode 100644 index 0000000000..aa55ebcd8f --- /dev/null +++ b/met/src/libcode/vx_nc_obs/nc_point_obs_in.h @@ -0,0 +1,58 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + +#ifndef __NC_POINT_OBS_IN_H__ +#define __NC_POINT_OBS_IN_H__ + +//////////////////////////////////////////////////////////////////////// + + +#include + +#include "observation.h" +#include "nc_utils.h" +#include "nc_obs_util.h" +#include "nc_var_info.h" +#include "nc_point_obs.h" + + +//////////////////////////////////////////////////////////////////////// + + +class MetNcPointObsIn : public MetNcPointObs { + + public: + + MetNcPointObsIn(); + ~MetNcPointObsIn(); + + bool check_nc(const char *nc_name, const char *caller=empty_name); + bool read_dim_headers(); + bool read_obs_data(); + bool read_obs_data(int buf_size, int start, float *obs_arr_block, + int *obs_qty_idx_block, char *obs_qty_str_block); + bool read_obs_data_numbers(); + bool read_obs_data_table_lookups(); + + // variables + + // data + +}; // MetNcPointObs + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __NC_POINT_OBS_IN_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_nc_obs/nc_point_obs_out.cc b/met/src/libcode/vx_nc_obs/nc_point_obs_out.cc new file mode 100644 index 0000000000..b758df0093 --- /dev/null +++ b/met/src/libcode/vx_nc_obs/nc_point_obs_out.cc @@ -0,0 +1,607 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +using namespace std; + +#include +#include +#include + +#include "vx_log.h" + +#include "nc_point_obs_out.h" +#include "nc_summary.h" +#include "write_netcdf.h" + +//////////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////////// + + + // + // Code for class MetNcPointObsOut + // + +//////////////////////////////////////////////////////////////////////// + +MetNcPointObsOut::MetNcPointObsOut() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +MetNcPointObsOut::~MetNcPointObsOut() { + close(); +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::init_from_scratch() { + MetNcPointObs::init_from_scratch(); + raw_hdr_cnt = 0; + reset_hdr_buffer = false; +} + +bool MetNcPointObsOut::add_header(const char *hdr_typ, const char *hdr_sid, + const time_t hdr_vld, const float hdr_lat, + const float hdr_lon, const float hdr_elv) +{ + bool added = false; + bool new_vld = false; + const char *method_name = "MetNcPointObsOut::add_header() "; + + // Can't filter duplicated one because header index was + // assigned before checking + int hdr_index; + add_header_strings(hdr_typ, hdr_sid); + + if (header_data.min_vld_time == -1 || header_data.min_vld_time > hdr_vld) { + if (header_data.min_vld_time == -1) header_data.max_vld_time = hdr_vld; + header_data.min_vld_time = hdr_vld; + new_vld = true; + } + else if (header_data.max_vld_time < hdr_vld) { + header_data.max_vld_time = hdr_vld; + new_vld = true; + } + if (new_vld || !header_data.vld_num_array.has(hdr_vld, hdr_index, false)) { + hdr_index = header_data.vld_array.n_elements(); + header_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); // Valid time + header_data.vld_num_array.add(hdr_vld); // Valid time + } + header_data.vld_idx_array.add(hdr_index); // Index of Valid time + + header_data.lat_array.add(hdr_lat); // Latitude + header_data.lon_array.add(hdr_lon); // Longitude + header_data.elv_array.add(hdr_elv); // Elevation + data_buffer.cur_hdr_idx++; + mlog << Debug(9) << method_name << "header is added (cur_hdr_idx=" + << data_buffer.cur_hdr_idx << ", obs_idx=" << data_buffer.cur_obs_idx << ")\n"; + added = true; + return added; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsOut::add_header_prepbufr(const int pb_report_type, + const int in_report_type, + const int instrument_type) +{ + bool added = true; + const char *method_name = "add_header_prepbufr() "; + // Can't filter duplicated one because header index was + // assigned before checking + header_data.prpt_typ_array.add(pb_report_type); + header_data.irpt_typ_array.add(in_report_type); + header_data.inst_typ_array.add(instrument_type); + mlog << Debug(9) << method_name << "pb_report_type: " << pb_report_type + << ", in_report_type: " << in_report_type << ", instrument_type: " + << instrument_type << "\n"; + return added; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsOut::add_header_strings(const char *hdr_typ, const char *hdr_sid) +{ + bool added = false; + + // Can't filter duplicated one because header index was + // assigned before checking + int hdr_index; + if (!header_data.typ_array.has(hdr_typ, hdr_index, false)) { + hdr_index = header_data.typ_array.n_elements(); + header_data.typ_array.add(hdr_typ); // Message type + added = true; + } + header_data.typ_idx_array.add(hdr_index); // Index of Message type + + if (!header_data.sid_array.has(hdr_sid, hdr_index, false)) { + hdr_index = header_data.sid_array.n_elements(); + header_data.sid_array.add(hdr_sid); // Station ID + added = true; + } + header_data.sid_idx_array.add(hdr_index); // Index of Station ID + + return added; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsOut::add_header_vld(const char *hdr_vld) +{ + bool added = false; + + // Can't filter duplicated one because header index was + // assigned before checking + int hdr_index; + if (!header_data.vld_array.has(hdr_vld, hdr_index, false)) { + hdr_index = header_data.typ_array.n_elements(); + header_data.vld_array.add(hdr_vld); // Valid time + added = true; + } + return added; +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::create_pb_hdrs(int pb_hdr_count) { + raw_hdr_cnt = pb_hdr_count; + obs_vars.create_pb_hdrs(obs_nc, pb_hdr_count); +} + +//////////////////////////////////////////////////////////////////////// +// If raw_hdr_cnt is greater than 0, skip updating header index for obs. + +void MetNcPointObsOut::get_dim_counts(int *obs_cnt, int *hdr_cnt) { + string method_name = "get_dim_counts() "; + SummaryObs *summary_obs = out_data.summary_obs; + bool do_summary = out_data.summary_info.flag; + + // + // Initialize the header and observation record counters + // + int obs_count = out_data.observations.size(); + int hdr_count = (out_data.processed_hdr_cnt > 0) + ? out_data.processed_hdr_cnt + : summary_obs->countHeaders(out_data.observations); // count and reset header index + if (do_summary) { + int summary_count = summary_obs->getSummaries().size(); + int summary_hdr_count = summary_obs->countSummaryHeaders(); + if (out_data.summary_info.raw_data) { + obs_count += summary_count; + hdr_count += summary_hdr_count; + } + else { + obs_count = summary_count; + hdr_count = summary_hdr_count; + if (out_data.processed_hdr_cnt > 0) { + reset_hdr_buffer = true; + } + } + } + *obs_cnt = obs_count; + *hdr_cnt = hdr_count; + mlog << Debug(7) << method_name << "obs_count: " + << obs_count << " header count: " << hdr_count << "\n"; + + // + // Add global attributes + // + + if (do_summary) write_summary_attributes(obs_nc, out_data.summary_info); + +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::init_buffer() { + + data_buffer.obs_data_idx = 0; + data_buffer.obs_data_offset = 0; + data_buffer.hdr_data_idx = 0; + data_buffer.hdr_data_offset = 0; + data_buffer.pb_hdr_data_offset = 0; + + strcpy(data_buffer.prev_hdr_typ_buf, "NotDefined"); + strcpy(data_buffer.prev_hdr_sid_buf, "NotDefined"); + strcpy(data_buffer.prev_hdr_vld_buf, "NotDefined"); + for (int index=0; indexgetName().c_str(), program_name.c_str()); + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::init_obs_vars(bool using_var_id, int deflate_level, + bool attr_agl) { + use_var_id = using_var_id; + obs_vars.reset(using_var_id); + obs_vars.attr_agl = attr_agl; + out_data.deflate_level = obs_vars.deflate_level = deflate_level; + + header_data.clear(); +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::reset_header_buffer(int buf_size, bool reset_all) { + for (int i=0; i observations, + SummaryObs *summary_obs, + TimeSummaryInfo summary_info, + int processed_hdr_cnt) { + out_data.processed_hdr_cnt = processed_hdr_cnt; + out_data.observations = observations; + out_data.summary_obs = summary_obs; + out_data.summary_info = summary_info; +} + +//////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::set_using_var_id(bool using_var_id) { + use_var_id = obs_vars.use_var_id = using_var_id; +} + +//////////////////////////////////////////////////////////////////////// +// Saves the headers at NcHeaderData header_data +// +void MetNcPointObsOut::write_arr_headers() { + int cur_hdr_idx = data_buffer.cur_hdr_idx; + int buf_size = (cur_hdr_idx > OBS_BUFFER_SIZE) ? OBS_BUFFER_SIZE : cur_hdr_idx; + const string method_name = " write_arr_headers()"; + + mlog << Debug(5) << method_name << " hdr_count: " << cur_hdr_idx + << ", typ_idx_array: " << header_data.typ_idx_array.n_elements() + << ", sid_idx_array: " << header_data.sid_idx_array.n_elements() + << ", vld_idx_array: " << header_data.vld_idx_array.n_elements() + << ", lat_array: " << header_data.lat_array.n_elements() + << ", lon_array: " << header_data.lon_array.n_elements() + << ", elv_array: " << header_data.elv_array.n_elements() + << "\n"; + + int hdr_data_idx = 0; + bool is_pb_hdr = (0 < header_data.prpt_typ_array.n_elements()) + && !IS_INVALID_NC(obs_vars.hdr_prpt_typ_var); + data_buffer.hdr_buf_size = buf_size; + data_buffer.hdr_data_idx = hdr_data_idx; + for (int index=0; index= buf_size) { + obs_vars.write_header_to_nc(data_buffer, hdr_data_idx, is_pb_hdr); + hdr_data_idx = data_buffer.hdr_data_idx; + } + } + + write_buf_headers(); +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::write_buf_headers () { + if (0 < data_buffer.hdr_data_idx) { + obs_vars.write_header_to_nc(data_buffer, data_buffer.hdr_data_idx); + } + obs_vars.write_table_vars(header_data, data_buffer); +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::write_header(const char *hdr_typ, const char *hdr_sid, + const time_t hdr_vld, const float hdr_lat, + const float hdr_lon, const float hdr_elv) +{ + // Can't filter duplicated one because header index was + // assigned before checking + int hdr_index; + bool new_vld = false; + int hdr_data_idx = data_buffer.hdr_data_idx; + const char *method_name = "MetNcPointObsOut::write_header"; + + // Message type + if (!header_data.typ_array.has(hdr_typ, hdr_index, false)) { + hdr_index = header_data.typ_array.n_elements(); + header_data.typ_array.add(hdr_typ); + } + data_buffer.hdr_typ_buf[hdr_data_idx] = hdr_index; + + // Station ID + if (!header_data.sid_array.has(hdr_sid, hdr_index, false)) { + hdr_index = header_data.sid_array.n_elements(); + header_data.sid_array.add(hdr_sid); + } + data_buffer.hdr_sid_buf[hdr_data_idx] = hdr_index; + + // Valid Time + if (header_data.min_vld_time == -1 || header_data.min_vld_time > hdr_vld) { + if (header_data.min_vld_time == -1) header_data.max_vld_time = hdr_vld; + header_data.min_vld_time = hdr_vld; + new_vld = true; + } + else if (header_data.max_vld_time < hdr_vld) { + header_data.max_vld_time = hdr_vld; + new_vld = true; + } + + if (new_vld || !header_data.vld_num_array.has(hdr_vld, hdr_index, false)) { + hdr_index = header_data.vld_array.n_elements(); + header_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); + header_data.vld_num_array.add(hdr_vld); + } + data_buffer.hdr_vld_buf[hdr_data_idx] = hdr_index; + + // Write the header array which consists of the following: + // LAT LON ELV + data_buffer.hdr_lat_buf[hdr_data_idx] = (float) hdr_lat; + data_buffer.hdr_lon_buf[hdr_data_idx] = (float) hdr_lon; + data_buffer.hdr_elv_buf[hdr_data_idx] = (float) hdr_elv; + + hdr_data_idx++; + data_buffer.hdr_data_idx = hdr_data_idx; + data_buffer.cur_hdr_idx++; + mlog << Debug(9) << method_name << "header is added (cur_index=" + << data_buffer.cur_hdr_idx << ")\n"; + + if (hdr_data_idx >= OBS_BUFFER_SIZE) { + obs_vars.write_header_to_nc(data_buffer, OBS_BUFFER_SIZE); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::write_observation() +{ + if (0 < data_buffer.obs_data_idx){ + obs_vars.write_obs_buffer(data_buffer, data_buffer.obs_data_idx); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::write_observation(const float obs_arr[OBS_ARRAY_LEN], + const char *obs_qty) +{ + int qty_index; + int obs_data_idx = data_buffer.obs_data_idx; + if (!data_buffer.qty_data_array.has(obs_qty, qty_index, false)) { + qty_index = data_buffer.qty_data_array.n_elements(); + data_buffer.qty_data_array.add(obs_qty); + } + data_buffer.qty_idx_buf[obs_data_idx] = qty_index; + + data_buffer.obs_hid_buf[obs_data_idx] = obs_arr[0]; + data_buffer.obs_vid_buf[obs_data_idx] = obs_arr[1]; + data_buffer.obs_lvl_buf[obs_data_idx] = obs_arr[2]; + data_buffer.obs_hgt_buf[obs_data_idx] = obs_arr[3]; + data_buffer.obs_val_buf[obs_data_idx] = obs_arr[4]; + data_buffer.obs_data_idx++; + data_buffer.cur_obs_idx++; + + if (data_buffer.obs_data_idx >= OBS_BUFFER_SIZE) { + obs_vars.write_obs_buffer(data_buffer, OBS_BUFFER_SIZE); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetNcPointObsOut::write_obs_data() +{ + string method_name = "write_obs_data() "; + bool do_summary = out_data.summary_info.flag; + bool do_save_raw_data = out_data.summary_info.raw_data; + bool do_header = (out_data.processed_hdr_cnt == 0); + + mlog << Debug(5) << method_name << "do_header: " << (do_header ? "true" : "false") + << ", do_summary: " << (do_summary ? "true" : "false") + << ", save_raw_data: " << (do_save_raw_data ? "true" : "false") + << "\n"; + + if (!do_summary || (do_summary && do_save_raw_data)) { + mlog << Debug(5) << method_name << "writing " + << (int)out_data.observations.size() << " raw data...\n"; + write_obs_data(out_data.observations, do_header); + } + if (do_summary) { + mlog << Debug(5) << method_name << "writing summary" + << (do_save_raw_data ? " " : " (summary only)") << "...\n"; + bool tmp_do_header = true ; + write_obs_data(out_data.summary_obs->getSummaries(), + tmp_do_header); + } + + int obs_buf_index = get_obs_index(); + if (obs_buf_index > 0) { + obs_vars.write_obs_buffer(data_buffer, obs_buf_index); + } + +} + +/////////////////////////////////////////////////////////////////////////////// + +int MetNcPointObsOut::write_obs_data(const vector< Observation > observations, + const bool do_header) +{ + int prev_hdr_idx = -1; + string prev_header_type = ""; + string prev_station_id = ""; + ConcatString obs_qty; + int headerOffset = data_buffer.cur_hdr_idx; + const string method_name = " write_obs_data()"; + + int obs_buf_size = observations.size(); + if (obs_buf_size > OBS_BUFFER_SIZE) obs_buf_size = OBS_BUFFER_SIZE; + + float obs_arr[OBS_ARRAY_LEN]; + bool header_to_vector = IS_INVALID_NC(obs_vars.hdr_arr_var) + || IS_INVALID_NC(obs_vars.hdr_lat_var); + mlog << Debug(5) << method_name << " obs_count: " << obs_buf_size + << " do_header: " << do_header + << " header_to_vector: " << header_to_vector << "\n"; + for (vector< Observation >::const_iterator obs = observations.begin(); + obs != observations.end(); ++obs) + { + data_buffer.processed_count++; + + if (do_header) { + if (obs->getHeaderIndex() != prev_hdr_idx) { + mlog << Debug(9) << method_name << " obs->getHeaderIndex(): " + << obs->getHeaderIndex() << " at obs " << data_buffer.processed_count << "\n"; + prev_hdr_idx = obs->getHeaderIndex(); + if (header_to_vector) { + add_header( + obs->getHeaderType().c_str(), + obs->getStationId().c_str(), + obs->getValidTime(), + obs->getLatitude(), + obs->getLongitude(), + obs->getElevation()); + } + else { + write_header( + obs->getHeaderType().c_str(), + obs->getStationId().c_str(), + obs->getValidTime(), + obs->getLatitude(), + obs->getLongitude(), + obs->getElevation()); + } + } + } + + obs_arr[0] = obs->getHeaderIndex(); + obs_arr[1] = (use_var_id ? obs->getVarCode() : obs->getGribCode()); + obs_arr[2] = obs->getPressureLevel(); + obs_arr[3] = obs->getHeight(); + obs_arr[4] = obs->getValue(); + obs_qty = (obs->getQualityFlag().length() == 0 ? na_str : obs->getQualityFlag().c_str()); + if (do_header) obs_arr[0] += headerOffset; + + write_observation(obs_arr, obs_qty.text()); + + } // endfor - obs + + if (data_buffer.obs_data_idx > 0) { + obs_vars.write_obs_buffer(data_buffer, data_buffer.obs_data_idx); + } + + //Caller handles writing headers + + return data_buffer.processed_count; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetNcPointObsOut::write_to_netcdf(StringArray obs_names, StringArray obs_units, + StringArray obs_descs) { + const char *method_name = " write_to_netcdf() "; + + write_obs_data(); + obs_vars.create_table_vars(obs_nc, header_data, data_buffer); + write_arr_headers(); + + if (use_var_id) { + int var_count = obs_names.n_elements(); + if (var_count > 0) { + int unit_count = obs_units.n(); + obs_vars.create_obs_name_vars (obs_nc, var_count, unit_count); + obs_vars.write_obs_var_names(obs_names); + if( unit_count > 0 ) obs_vars.write_obs_var_units(obs_units); + if( obs_descs.n() > 0 ) obs_vars.write_obs_var_descriptions(obs_descs); + mlog << Debug(7) << method_name << var_count + << " variable names were saved\n"; + } + else mlog << Warning << "\n" << method_name + << "variable names are not added because of empty names\n\n"; + } + else mlog << Debug(7) << method_name << "use_var_id is false\n"; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_nc_obs/nc_point_obs_out.h b/met/src/libcode/vx_nc_obs/nc_point_obs_out.h new file mode 100644 index 0000000000..510aadd767 --- /dev/null +++ b/met/src/libcode/vx_nc_obs/nc_point_obs_out.h @@ -0,0 +1,104 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + +#ifndef __NC_POINT_OBS_OUT_H__ +#define __NC_POINT_OBS_OUT_H__ + +//////////////////////////////////////////////////////////////////////// + +#include + +#include "observation.h" +#include "nc_utils.h" +#include "nc_obs_util.h" +#include "nc_point_obs.h" +#include "nc_var_info.h" +#include "vx_summary.h" + + +//////////////////////////////////////////////////////////////////////// + +class MetNcPointObsOut : public MetNcPointObs { + + protected: + int raw_hdr_cnt; + bool reset_hdr_buffer; + + NcDataBuffer data_buffer; + NcObsOutputData out_data; + + void init_from_scratch(); + + public: + + MetNcPointObsOut(); + ~MetNcPointObsOut(); + + bool add_header(const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, + const float hdr_lat, const float hdr_lon, const float hdr_elv); + bool add_header_prepbufr (const int pb_report_type, const int in_report_type, + const int instrument_type); + bool add_header_strings(const char *hdr_typ, const char *hdr_sid); + bool add_header_vld(const char *hdr_vld); + + void create_pb_hdrs(int pb_hdr_count); + + int get_buf_size(); + void get_dim_counts(int *obs_cnt, int *hdr_cnt); + int get_hdr_index(); + int get_obs_index(); + NcObsOutputData *get_output_data(); + NetcdfObsVars *get_obs_vars(); + + void init_buffer(); + void init_obs_vars(bool using_var_id, int deflate_level, bool attr_agl=false); + bool init_netcdf(int obs_count, int hdr_count, string program_name); + + void reset_header_buffer(int buf_size, bool reset_all); + void set_nc_out_data(vector observations, + SummaryObs *summary_obs, TimeSummaryInfo summary_info, + int processed_hdr_cnt=0); + void set_using_var_id(bool using_var_id); + + void write_arr_headers(); + void write_buf_headers (); + void write_header (const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, + const float hdr_lat, const float hdr_lon, const float hdr_elv); + void write_observation(); + void write_observation(const float obs_arr[OBS_ARRAY_LEN], const char *obs_qty); + void write_obs_data(); + int write_obs_data(const vector< Observation > observations, + const bool do_header = true); + bool write_to_netcdf(StringArray obs_names, StringArray obs_units, + StringArray obs_descs); + + // variables + + // data + +}; // MetNcPointObsOut + + +//////////////////////////////////////////////////////////////////////// + +inline int MetNcPointObsOut::get_hdr_index() { return data_buffer.cur_hdr_idx; } +inline int MetNcPointObsOut::get_obs_index() { return data_buffer.obs_data_idx; } +inline NcObsOutputData *MetNcPointObsOut::get_output_data() { return &out_data; } +inline NetcdfObsVars *MetNcPointObsOut::get_obs_vars() { return &obs_vars; } + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __NC_POINT_OBS_OUT_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_nc_obs/nc_summary.cc b/met/src/libcode/vx_nc_obs/nc_summary.cc index 2870ee713c..b7c2786027 100644 --- a/met/src/libcode/vx_nc_obs/nc_summary.cc +++ b/met/src/libcode/vx_nc_obs/nc_summary.cc @@ -8,7 +8,7 @@ //////////////////////////////////////////////////////////////////////// // -// Filename: summary_util.cc +// Filename: nc_summary.cc // // Description: // Common routines for time summary (into NetCDF). @@ -16,21 +16,16 @@ using namespace std; -//#include #include -//#include "vx_math.h" -//#include "vx_nc_util.h" #include "write_netcdf.h" #include "nc_obs_util.h" -#include "nc_summary.h" +#include "vx_summary.h" //////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////// - -string _secsToTimeString(const int secs) +string seconds_to_time_string(const int secs) { // Get the different fields from the number of seconds @@ -50,108 +45,13 @@ string _secsToTimeString(const int secs) return string(string_buffer); } -//////////////////////////////////////////////////////////////////////// -// If raw_hdr_cnt is greater than 0, skip updating header index for obs. - -void init_netcdf_output(NcFile *nc_file, NetcdfObsVars &obs_vars, - NcObsOutputData &nc_out_data, string program_name) -{ - string method_name = "init_netcdf_output() "; - SummaryObs *summary_obs = nc_out_data.summary_obs; - bool do_summary = nc_out_data.summary_info.flag; - - // - // Initialize the header and observation record counters - // - int obs_count = nc_out_data.observations.size(); - int hdr_count = (nc_out_data.processed_hdr_cnt > 0) - ? nc_out_data.processed_hdr_cnt - : summary_obs->countHeaders(nc_out_data.observations); // count and reset header index - if (do_summary) { - int summary_count = summary_obs->getSummaries().size(); - int summary_hdr_count = summary_obs->countSummaryHeaders(); - if (nc_out_data.summary_info.raw_data) { - obs_count += summary_count; - hdr_count += summary_hdr_count; - } - else { - if (nc_out_data.processed_hdr_cnt > 0) { - bool hdr_cnt = get_nc_hdr_cur_index(); - bool reset_array = true; - reset_header_buffer(hdr_cnt, reset_array); - mlog << Debug(5) << method_name << "reset headers (" << hdr_cnt << ") raw data.\n"; - } - obs_count = summary_count; - hdr_count = summary_hdr_count; - } - } - mlog << Debug(7) << method_name << "obs_count: " - << obs_count << " header count: " << hdr_count << "\n"; - obs_vars.obs_cnt = obs_count; - - create_nc_hdr_vars(obs_vars, nc_file, hdr_count, nc_out_data.deflate_level); - create_nc_obs_vars(obs_vars, nc_file, nc_out_data.deflate_level, - obs_vars.use_var_id); - - // - // Add global attributes - // - write_netcdf_global(nc_file, nc_file->getName().c_str(), program_name.c_str()); - - if (do_summary) write_summary_attributes(nc_file, nc_out_data.summary_info); - -} - -//////////////////////////////////////////////////////////////////////// - -bool write_observations(NcFile *nc_file, NetcdfObsVars &obs_vars, - NcObsOutputData &nc_out_data) -{ - bool use_var_id = obs_vars.use_var_id; - string method_name = "write_observations() "; - bool do_summary = nc_out_data.summary_info.flag; - bool do_save_raw_data = nc_out_data.summary_info.raw_data; - bool do_header = (nc_out_data.processed_hdr_cnt == 0); - - mlog << Debug(5) << method_name << "do_header: " << (do_header ? "true" : "false") - << ", do_summary: " << (do_summary ? "true" : "false") - << ", save_raw_data: " << (do_save_raw_data ? "true" : "false") - << "\n"; - - if (!do_summary || (do_summary && do_save_raw_data)) { - mlog << Debug(5) << method_name << "writing " - << (int)nc_out_data.observations.size() << " raw data...\n"; - write_nc_observations(obs_vars, nc_out_data.observations, use_var_id, do_header); - } - if (do_summary) { - mlog << Debug(5) << method_name << "writing summary" - << (do_save_raw_data ? " " : " (summary only)") << "...\n"; - bool tmp_do_header = true; - write_nc_observations(obs_vars, nc_out_data.summary_obs->getSummaries(), - use_var_id, tmp_do_header); - } - - int obs_buf_index = get_nc_obs_buf_index(); - if (obs_buf_index > 0) { - write_nc_obs_buffer(obs_buf_index); - } - - create_nc_table_vars (obs_vars, nc_file, nc_out_data.deflate_level); - - write_nc_arr_headers(obs_vars); - - //write_nc_table_vars(obs_vars); - - return true; -} - //////////////////////////////////////////////////////////////////////// void write_summary_attributes(NcFile *nc_file, TimeSummaryInfo summary_info) { add_att(nc_file, "time_summary_beg", - _secsToTimeString(summary_info.beg)); + seconds_to_time_string(summary_info.beg)); add_att(nc_file, "time_summary_end", - _secsToTimeString(summary_info.end)); + seconds_to_time_string(summary_info.end)); char att_string[1024]; @@ -187,4 +87,3 @@ void write_summary_attributes(NcFile *nc_file, TimeSummaryInfo summary_info) { } //////////////////////////////////////////////////////////////////////// - diff --git a/met/src/libcode/vx_nc_obs/nc_summary.h b/met/src/libcode/vx_nc_obs/nc_summary.h index 85d11ec249..3f42868ff4 100644 --- a/met/src/libcode/vx_nc_obs/nc_summary.h +++ b/met/src/libcode/vx_nc_obs/nc_summary.h @@ -16,6 +16,7 @@ #include using namespace netCDF; #include "nc_utils.h" +#include "nc_obs_util.h" #include "vx_summary.h" //////////////////////////////////////////////////////////////////////// @@ -33,10 +34,7 @@ struct NcObsOutputData { extern string _secsToTimeString(const int secs); -extern void init_netcdf_output(NcFile *, NetcdfObsVars &obs_vars, - NcObsOutputData &nc_out_data, string program_name); - -extern bool write_observations(NcFile *, NetcdfObsVars &, NcObsOutputData &nc_out_data); +// Not moved to nc_obs_util to reduce the dependency (library) extern void write_summary_attributes(NcFile *, TimeSummaryInfo); diff --git a/met/src/libcode/vx_nc_util/nc_utils.h b/met/src/libcode/vx_nc_util/nc_utils.h index e1e99d6c10..f1dec583e3 100644 --- a/met/src/libcode/vx_nc_util/nc_utils.h +++ b/met/src/libcode/vx_nc_util/nc_utils.h @@ -77,6 +77,7 @@ static const string C_unknown_str = string("unknown"); #define NC_BUFFER_SIZE_16K (16*1024) #define HDR_ARRAY_LEN 3 // Observation header length +#define HDR_TYPE_ARR_LEN 3 // Observation header type length (prpt/irpt/inst) #define OBS_ARRAY_LEN 5 // Observation values length #define HEADER_STR_LEN 16 // Maximum length for header string #define HEADER_STR_LEN2 40 // Maximum length for header string 2 diff --git a/met/src/libcode/vx_nc_util/write_netcdf.cc b/met/src/libcode/vx_nc_util/write_netcdf.cc index bf8eff148c..71d67342b7 100644 --- a/met/src/libcode/vx_nc_util/write_netcdf.cc +++ b/met/src/libcode/vx_nc_util/write_netcdf.cc @@ -315,44 +315,3 @@ ConcatString s; /////////////////////////////////////////////////////////////////////////////// -//bool is_same_header (const char *hdr_typ, const char *hdr_sid, const char *hdr_vld, -// const float hdr_lat, const float hdr_lon, const float hdr_elv) { -// bool new_header = -// !is_eq(nc_data_buffer.prev_hdr_arr_buf[0],hdr_lat) || -// !is_eq(nc_data_buffer.prev_hdr_arr_buf[1], hdr_lon) || -// !is_eq(nc_data_buffer.prev_hdr_arr_buf[2], hdr_elv) || -// 0 != strcmp(nc_data_buffer.prev_hdr_typ_buf, hdr_typ) || -// 0 != strcmp(nc_data_buffer.prev_hdr_sid_buf, hdr_sid) || -// 0 != strcmp(nc_data_buffer.prev_hdr_vld_buf, hdr_vld); -// if (new_header) { -// strcpy(nc_data_buffer.prev_hdr_typ_buf, hdr_typ); -// strcpy(nc_data_buffer.prev_hdr_sid_buf, hdr_sid); -// strcpy(nc_data_buffer.prev_hdr_vld_buf, hdr_vld); -// nc_data_buffer.prev_hdr_arr_buf[0] = hdr_lat; -// nc_data_buffer.prev_hdr_arr_buf[1] = hdr_lon; -// nc_data_buffer.prev_hdr_arr_buf[2] = hdr_elv; -// } -// return new_header; -//} -// -//bool is_same_header (const char *hdr_typ, const char *hdr_sid, const unixtime hdr_vld, -// const float hdr_lat, const float hdr_lon, const float hdr_elv) { -// bool new_header = -// !is_eq(nc_data_buffer.prev_hdr_arr_buf[0],hdr_lat) || -// !is_eq(nc_data_buffer.prev_hdr_arr_buf[1], hdr_lon) || -// !is_eq(nc_data_buffer.prev_hdr_arr_buf[2], hdr_elv) || -// 0 != strcmp(nc_data_buffer.prev_hdr_typ_buf, hdr_typ) || -// 0 != strcmp(nc_data_buffer.prev_hdr_sid_buf, hdr_sid) || -// !is_eq(nc_data_buffer.prev_hdr_vld, hdr_vld); -// if (new_header) { -// strcpy(nc_data_buffer.prev_hdr_typ_buf, hdr_typ); -// strcpy(nc_data_buffer.prev_hdr_sid_buf, hdr_sid); -// nc_data_buffer.prev_hdr_vld = hdr_vld; -// nc_data_buffer.prev_hdr_arr_buf[0] = hdr_lat; -// nc_data_buffer.prev_hdr_arr_buf[1] = hdr_lon; -// nc_data_buffer.prev_hdr_arr_buf[2] = hdr_elv; -// } -// return new_header; -//} - -/////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/core/ensemble_stat/Makefile.am b/met/src/tools/core/ensemble_stat/Makefile.am index 172302f19a..5aec02dd98 100644 --- a/met/src/tools/core/ensemble_stat/Makefile.am +++ b/met/src/tools/core/ensemble_stat/Makefile.am @@ -24,20 +24,20 @@ ensemble_stat_LDADD = -lvx_stat_out \ -lvx_data2d_nc_met \ -lvx_data2d_grib $(GRIB2_LIBS) \ -lvx_data2d_nc_pinterp \ - $(PYTHON_LIBS) \ -lvx_data2d_nccf \ - -lvx_statistics \ + $(PYTHON_LIBS) \ -lvx_data2d \ -lvx_nc_obs \ -lvx_nc_util \ -lvx_regrid \ -lvx_grid \ -lvx_config \ - -lvx_cal \ + -lvx_color \ -lvx_util \ -lvx_math \ - -lvx_color \ + -lvx_cal \ -lvx_log \ + $(PYTHON_LIBS) \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas EXTRA_DIST = ensemble_stat.h \ diff --git a/met/src/tools/core/ensemble_stat/ensemble_stat.cc b/met/src/tools/core/ensemble_stat/ensemble_stat.cc index b3e736fdae..eecc5c183f 100644 --- a/met/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/met/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -86,6 +86,7 @@ using namespace std; #include "vx_log.h" #include "nc_obs_util.h" +#include "nc_point_obs_in.h" //////////////////////////////////////////////////////////////////////// @@ -941,124 +942,60 @@ void process_point_obs(int i_nc) { int i_obs, j; unixtime hdr_ut; NcFile *obs_in = (NcFile *) 0; + const char *method_name = "process_point_obs() -> "; mlog << Debug(2) << "\n" << sep_str << "\n\n" << "Processing point observation file: " << point_obs_file_list[i_nc] << "\n"; // Open the observation file as a NetCDF file. - obs_in = open_ncfile(point_obs_file_list[i_nc].c_str()); + MetNcPointObsIn nc_point_obs; + if(!nc_point_obs.open(point_obs_file_list[i_nc].c_str())) { + nc_point_obs.close(); - if(IS_INVALID_NC_P(obs_in)) { - delete obs_in; - obs_in = (NcFile *) 0; - - mlog << Warning << "\nprocess_point_obs() -> " + mlog << Warning << "\n" << method_name << "can't open observation netCDF file: " << point_obs_file_list[i_nc] << "\n\n"; return; } // Read the dimensions and variables - NetcdfObsVars obs_vars; - read_nc_dims_vars(obs_vars, obs_in); - - bool use_var_id = obs_vars.use_var_id; - if (use_var_id) { - NcDim var_dim = get_nc_dim(obs_in,nc_dim_nvar); - get_dim_size(&var_dim); - } + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(point_obs_file_list[i_nc].c_str(), method_name); // exit if missing dims/vars + nc_point_obs.read_obs_data_table_lookups(); - int exit_code = check_nc_dims_vars(obs_vars); + int hdr_count = nc_point_obs.get_hdr_cnt(); + int obs_count = nc_point_obs.get_obs_cnt(); + bool use_var_id = nc_point_obs.is_using_var_id(); + bool use_arr_vars = nc_point_obs.is_using_obs_arr(); - if(exit_code == exit_code_no_dim) { - mlog << Error << "\nprocess_point_obs() -> " - << "can't read \"mxstr\", \"nobs\" or \"nmsg\" " - << "dimensions from netCDF file: " - << point_obs_file_list[i_nc] << "\n\n"; - exit(1); - } - - // Read the variables - - if(exit_code == exit_code_no_hdr_vars) { - mlog << Error << "\nprocess_point_obs() -> " - << "can't read \"hdr_typ\", \"hdr_sid\", " - << "or \"hdr_vld\" variables from netCDF file: " - << point_obs_file_list[i_nc] << "\n\n"; - exit(1); - } - if(exit_code == exit_code_no_loc_vars) { - mlog << Error << "\nprocess_point_obs() -> " - << "can't read \"hdr_arr\", or \"hdr_lat\" variables from netCDF file: " - << point_obs_file_list[i_nc] << "\n\n"; - exit(1); - } - if(exit_code == exit_code_no_obs_vars) { - mlog << Error << "\nprocess_point_obs() -> " - << "can't read \"obs_arr\" or \"obs_val\" variables from netCDF file: " - << point_obs_file_list[i_nc] << "\n\n"; - exit(1); - } - - if(IS_INVALID_NC(obs_vars.obs_qty_var)) - mlog << Debug(3) << "Quality marker information not found input file.\n"; - - bool use_arr_vars = !IS_INVALID_NC(obs_vars.obs_arr_var); - int hdr_count = GET_NC_SIZE(obs_vars.hdr_dim); - int obs_count = GET_NC_SIZE(obs_vars.obs_dim); mlog << Debug(2) << "Searching " << (obs_count) << " observations from " << (hdr_count) << " header messages.\n"; - int qty_len = get_nc_string_length(obs_in, obs_vars.obs_qty_tbl_var, - (use_arr_vars ? nc_var_obs_qty : nc_var_obs_qty_tbl)); - - NcHeaderData header_data = get_nc_hdr_data(obs_vars); - int buf_size = ((obs_count > DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : (obs_count)); int obs_qty_idx_block[buf_size]; float obs_arr_block[buf_size][OBS_ARRAY_LEN]; - char obs_qty_str_block[buf_size][qty_len]; - StringArray obs_qty_array; - - float obs_arr[OBS_ARRAY_LEN], hdr_arr[hdr_arr_len]; - int hdr_typ_arr[hdr_typ_arr_len]; + float obs_arr[OBS_ARRAY_LEN], hdr_arr[HDR_ARRAY_LEN]; + int hdr_typ_arr[HDR_TYPE_ARR_LEN]; ConcatString hdr_typ_str; ConcatString hdr_sid_str; ConcatString hdr_vld_str; ConcatString obs_qty_str; - - StringArray var_names; ConcatString var_name; - - if (use_var_id) { - NcVar obs_var = get_nc_var(obs_in, nc_var_obs_var); - if (!get_nc_data_to_array(&obs_var, &var_names)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting variable names from " - << GET_NC_NAME(obs_var) << "\n\n"; - exit(1); - } - } - - if (!IS_INVALID_NC(obs_vars.obs_qty_tbl_var)) { - if (!get_nc_data_to_array(&obs_vars.obs_qty_tbl_var, &obs_qty_array)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_qty\n\n"; - exit(1); - } - } + StringArray var_names; + StringArray obs_qty_array = nc_point_obs.get_qty_data(); + if(use_var_id) var_names = nc_point_obs.get_var_names(); for(int i_start=0; i_start DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : (obs_count-i_start); + buf_size = ((obs_count-i_start) > DEF_NC_BUFFER_SIZE) + ? DEF_NC_BUFFER_SIZE : (obs_count-i_start); - if (!read_nc_obs_data(obs_vars, buf_size, i_start, qty_len, - (float *)obs_arr_block, obs_qty_idx_block, (char *)obs_qty_str_block)) { + if (!nc_point_obs.read_obs_data(buf_size, i_start, (float *)obs_arr_block, + obs_qty_idx_block, (char *)0)) { exit(1); } - // Process each observation in the file for(int i_offset=0; i_offset= hdr_count) { - mlog << Warning << "\nprocess_point_obs() -> " + mlog << Warning << "\n" << method_name << "range check error for header index " << headerOffset << " from observation number " << i_obs << " of point observation file: " @@ -1088,35 +1021,19 @@ void process_point_obs(int i_nc) { } // Read the corresponding header array for this observation - hdr_arr[0] = header_data.lat_array[headerOffset]; - hdr_arr[1] = header_data.lon_array[headerOffset]; - hdr_arr[2] = header_data.elv_array[headerOffset]; + // - the corresponding header type, header Station ID, and valid time + nc_point_obs.get_header(headerOffset, hdr_arr, hdr_typ_str, + hdr_sid_str, hdr_vld_str); // Read the header integer types - hdr_typ_arr[0] = (header_data.prpt_typ_array.n() > headerOffset ? - header_data.prpt_typ_array[headerOffset] : bad_data_int); - hdr_typ_arr[1] = (header_data.irpt_typ_array.n() > headerOffset ? - header_data.irpt_typ_array[headerOffset] : bad_data_int); - hdr_typ_arr[2] = (header_data.inst_typ_array.n() > headerOffset ? - header_data.inst_typ_array[headerOffset] : bad_data_int); - - // Read the corresponding header type for this observation - hdr_idx = use_arr_vars ? headerOffset : header_data.typ_idx_array[headerOffset]; - hdr_typ_str = header_data.typ_array[hdr_idx]; - - // Read the corresponding header Station ID for this observation - hdr_idx = use_arr_vars ? headerOffset : header_data.sid_idx_array[headerOffset]; - hdr_sid_str = header_data.sid_array[hdr_idx]; - - // Read the corresponding valid time for this observation - hdr_idx = use_arr_vars ? headerOffset : header_data.vld_idx_array[headerOffset]; - hdr_vld_str = header_data.vld_array[hdr_idx]; + nc_point_obs.get_header_type(headerOffset, hdr_typ_arr); // Convert string to a unixtime hdr_ut = timestring_to_unix(hdr_vld_str.c_str()); - if (use_var_id && obs_arr[1] < var_names.n()) { - var_name = var_names[obs_arr[1]]; + int grib_code = nc_point_obs.get_grib_code_or_var_index(obs_arr); + if (use_var_id && grib_code < var_names.n()) { + var_name = var_names[grib_code]; } else { var_name = ""; @@ -1127,21 +1044,17 @@ void process_point_obs(int i_nc) { for(j=0; j " + mlog << Warning << "\n" << method_name << "can't open observation netCDF file: " << obs_file[i_nc] << "\n\n"; return; } - // Read the dimensions and variables - NetcdfObsVars obs_vars; - read_nc_dims_vars(obs_vars, obs_in); - - bool use_var_id = obs_vars.use_var_id; - if (use_var_id) { - NcDim var_dim = get_nc_dim(obs_in,nc_dim_nvar); - get_dim_size(&var_dim); - } - - int exit_code = check_nc_dims_vars(obs_vars); - if(exit_code == exit_code_no_dim) { - mlog << Error << "\nprocess_obs_file() -> " - << "can't read \"mxstr\", \"nobs\" or \"nmsg\" " - << "dimensions from netCDF file: " - << obs_file[i_nc] << "\n\n"; - exit(1); - } - - if(exit_code == exit_code_no_hdr_vars) { - mlog << Error << "\nprocess_obs_file() -> " - << "can't read \"hdr_typ\", \"hdr_sid\", " - << "or \"hdr_vld\" variables from netCDF file: " - << obs_file[i_nc] << "\n\n"; - exit(1); - } - - if(exit_code == exit_code_no_loc_vars) { - mlog << Error << "\nprocess_obs_file() -> " - << "can't read \"hdr_arr\" or \"hdr_lat\" " - << "variables from netCDF file: " - << obs_file[i_nc] << "\n\n"; - exit(1); - } - if(exit_code == exit_code_no_obs_vars) { - mlog << Error << "\nprocess_obs_file() -> " - << "can't read \"obs_arr\" or \"obs_val\" " - << "variables from netCDF file: " - << obs_file[i_nc] << "\n\n"; - exit(1); - } - - if(IS_INVALID_NC(obs_vars.obs_qty_var)) - mlog << Debug(3) << "Quality marker information not found input file\n"; - - int obs_count = get_dim_size(&obs_vars.obs_dim); - int hdr_count = get_dim_size(&obs_vars.hdr_dim); + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(obs_file[i_nc].c_str(), method_name); + nc_point_obs.read_obs_data_table_lookups(); + bool use_var_id = nc_point_obs.is_using_var_id(); + int hdr_count = nc_point_obs.get_hdr_cnt(); + int obs_count = nc_point_obs.get_obs_cnt(); mlog << Debug(2) << "Searching " << obs_count << " observations from " << hdr_count << " messages.\n"; - StringArray var_names; ConcatString var_name(""); - if (use_var_id) { - if (!get_nc_data_to_array(obs_in, nc_var_obs_var, &var_names)) { - mlog << Error << "\nprocess_obs_file() -> " - << "trouble getting variable names from " - << nc_var_obs_var << "\n\n"; - exit(1); - } - } + bool use_arr_vars = nc_point_obs.is_using_obs_arr(); + StringArray var_names; + StringArray obs_qty_array = nc_point_obs.get_qty_data(); + if( use_var_id ) var_names = nc_point_obs.get_var_names(); - bool use_arr_vars = !IS_INVALID_NC(obs_vars.obs_arr_var); int buf_size = ((obs_count > BUFFER_SIZE) ? BUFFER_SIZE : (obs_count)); - NcHeaderData header_data = get_nc_hdr_data(obs_vars); - int typ_len = header_data.typ_len; - int sid_len = header_data.sid_len; - int vld_len = header_data.vld_len; - int qty_len = get_nc_string_length(obs_in, obs_vars.obs_qty_tbl_var, - (use_arr_vars ? nc_var_obs_qty : nc_var_obs_qty_tbl)); - int obs_qty_idx_block[buf_size]; float obs_arr_block[buf_size][OBS_ARRAY_LEN]; - char obs_qty_block[buf_size][qty_len]; - StringArray obs_qty_array; - - if (!IS_INVALID_NC(obs_vars.obs_qty_tbl_var)) { - if (!get_nc_data_to_array(&obs_vars.obs_qty_tbl_var, &obs_qty_array)) { - mlog << Error << "\nprocess_obs_file() -> " - << "trouble getting obs_qty\n\n"; - exit(1); - } - } // Process each observation in the file int str_length, block_size; @@ -764,13 +700,13 @@ void process_obs_file(int i_nc) { block_size = (obs_count - i_block_start_idx); if (block_size > BUFFER_SIZE) block_size = BUFFER_SIZE; - if (!read_nc_obs_data(obs_vars, block_size, i_block_start_idx, qty_len, - (float *)obs_arr_block, obs_qty_idx_block, (char *)obs_qty_block)) { + if (!nc_point_obs.read_obs_data(block_size, i_block_start_idx, + (float *)obs_arr_block, + obs_qty_idx_block, (char *)0)) { exit(1); } int hdr_idx; - strcpy(obs_qty_str, ""); for(int i_block_idx=0; i_block_idx= hdr_count) { - mlog << Warning << "\nprocess_obs_file() -> " + mlog << Warning << "\n" << method_name << "range check error for header index " << headerOffset << " from observation number " << i_obs << " of point observation file: " << obs_file[i_nc] @@ -798,46 +730,26 @@ void process_obs_file(int i_nc) { } // Read the corresponding header array for this observation - hdr_arr[0] = header_data.lat_array[headerOffset]; - hdr_arr[1] = header_data.lon_array[headerOffset]; - hdr_arr[2] = header_data.elv_array[headerOffset]; - - // Read the corresponding header type for this observation - hdr_idx = use_arr_vars ? headerOffset : header_data.typ_idx_array[headerOffset]; - str_length = header_data.typ_array[hdr_idx].length(); - if (str_length > typ_len) str_length = typ_len; - strncpy(hdr_typ_str, header_data.typ_array[hdr_idx].c_str(), str_length); - hdr_typ_str[str_length] = bad_data_char; - - // Read the corresponding header Station ID for this observation - hdr_idx = use_arr_vars ? headerOffset : header_data.sid_idx_array[headerOffset]; - str_length = header_data.sid_array[hdr_idx].length(); - if (str_length > sid_len) str_length = sid_len; - strncpy(hdr_sid_str, header_data.sid_array[hdr_idx].c_str(), str_length); - hdr_sid_str[str_length] = bad_data_char; - - // Read the corresponding valid time for this observation - hdr_idx = use_arr_vars ? headerOffset : header_data.vld_idx_array[headerOffset]; - str_length = header_data.vld_array[hdr_idx].length(); - if (str_length > vld_len) str_length = vld_len; - strncpy(hdr_vld_str, header_data.vld_array[hdr_idx].c_str(), str_length); - hdr_vld_str[str_length] = bad_data_char; + // - the corresponding header type, header Station ID, and valid time + nc_point_obs.get_header(headerOffset, hdr_arr, hdr_typ_str, + hdr_sid_str, hdr_vld_str); // Store the variable name - int grib_code = obs_arr[1]; + int org_grib_code = nc_point_obs.get_grib_code_or_var_index(obs_arr); + int grib_code = org_grib_code; if (use_var_id && grib_code < var_names.n()) { var_name = var_names[grib_code]; - obs_arr[1] = bad_data_int; + grib_code = bad_data_int; } else { var_name = ""; } // Check for wind components - is_ugrd = ( use_var_id && var_name == ugrd_abbr_str ) || - (!use_var_id && nint(obs_arr[1]) == ugrd_grib_code); - is_vgrd = ( use_var_id && var_name == vgrd_abbr_str ) || - (!use_var_id && nint(obs_arr[1]) == vgrd_grib_code); + is_ugrd = ( use_var_id && var_name == ugrd_abbr_str ) || + (!use_var_id && nint(grib_code) == ugrd_grib_code); + is_vgrd = ( use_var_id && var_name == vgrd_abbr_str ) || + (!use_var_id && nint(grib_code) == vgrd_grib_code); // If the current observation is UGRD, save it as the // previous. If vector winds are to be computed, UGRD @@ -852,10 +764,8 @@ void process_obs_file(int i_nc) { // and at the same vertical level. if(vflag && is_vgrd) { - if(!is_eq(obs_arr[0], prev_obs_arr[0]) || - !is_eq(obs_arr[2], prev_obs_arr[2]) || - !is_eq(obs_arr[3], prev_obs_arr[3])) { - mlog << Error << "\nprocess_obs_file() -> " + if(!nc_point_obs.is_same_obs_values(obs_arr, prev_obs_arr)) { + mlog << Error << "\n" << method_name << "for observation index " << i_obs << ", when computing VL1L2 and/or VAL1L2 vector winds " << "each UGRD observation must be followed by a VGRD " @@ -866,7 +776,7 @@ void process_obs_file(int i_nc) { } // Convert string to a unixtime - hdr_ut = timestring_to_unix(hdr_vld_str); + hdr_ut = timestring_to_unix(hdr_vld_str.c_str()); // Check each conf_info.vx_pd object to see if this observation // should be added @@ -876,23 +786,19 @@ void process_obs_file(int i_nc) { if(conf_info.vx_opt[j].vx_pd.fcst_dpa.n_planes() == 0) continue; // Attempt to add the observation to the conf_info.vx_pd object - conf_info.vx_opt[j].vx_pd.add_point_obs(hdr_arr, - hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty_str, obs_arr, - grid, var_name.c_str()); + conf_info.vx_opt[j].vx_pd.add_point_obs( + hdr_arr, hdr_typ_str.c_str(), hdr_sid_str.c_str(), + hdr_ut, obs_qty_str.c_str(), obs_arr, + grid, var_name.c_str()); } - obs_arr[1] = grib_code; + nc_point_obs.set_grib_code_or_var_index(obs_arr, org_grib_code); } } // end for i_block_start_idx // Deallocate and clean up - if(obs_in) { - delete obs_in; - obs_in = (NcFile *) 0; - } - clear_header_data(&header_data); + nc_point_obs.close(); return; } diff --git a/met/src/tools/core/point_stat/point_stat.h b/met/src/tools/core/point_stat/point_stat.h index 9d34a7282a..dc232a52de 100644 --- a/met/src/tools/core/point_stat/point_stat.h +++ b/met/src/tools/core/point_stat/point_stat.h @@ -63,12 +63,6 @@ static const char * program_name = "point_stat"; static const char * default_config_filename = "MET_BASE/config/PointStatConfig_default"; -// Observation header length -static const int hdr_arr_len = 3; - -// Observation values length -static const int obs_arr_len = 5; - // Header columns static const char **txt_columns[n_txt] = { fho_columns, ctc_columns, cts_columns, diff --git a/met/src/tools/other/ascii2nc/Makefile.am b/met/src/tools/other/ascii2nc/Makefile.am index 7fce7b81ba..f548cb9330 100644 --- a/met/src/tools/other/ascii2nc/Makefile.am +++ b/met/src/tools/other/ascii2nc/Makefile.am @@ -31,7 +31,6 @@ ascii2nc_LDFLAGS = ${MET_LDFLAGS} ascii2nc_LDADD = -lvx_stat_out \ -lvx_statistics \ -lvx_shapedata \ - -lvx_gsl_prob \ -lvx_analysis_util \ -lvx_data2d_factory \ -lvx_data2d_nc_met \ @@ -39,22 +38,21 @@ ascii2nc_LDADD = -lvx_stat_out \ -lvx_data2d_nc_pinterp \ -lvx_data2d_nccf \ $(PYTHON_LIBS) \ - -lvx_statistics \ -lvx_data2d \ -lvx_nc_obs \ -lvx_nc_util \ -lvx_regrid \ -lvx_grid \ + -lvx_summary \ + -lvx_pb_util \ -lvx_config \ -lvx_gsl_prob \ - -lvx_pb_util \ - -lvx_cal \ + -lvx_color \ -lvx_util \ - $(PYTHON_LIBS) \ + -lvx_cal \ -lvx_math \ - -lvx_color \ -lvx_log \ - -lvx_summary \ + $(PYTHON_LIBS) \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas diff --git a/met/src/tools/other/ascii2nc/file_handler.cc b/met/src/tools/other/ascii2nc/file_handler.cc index f5b57aec74..106667f8d3 100644 --- a/met/src/tools/other/ascii2nc/file_handler.cc +++ b/met/src/tools/other/ascii2nc/file_handler.cc @@ -76,7 +76,7 @@ FileHandler::~FileHandler() bool FileHandler::readAsciiFiles(const vector< ConcatString > &ascii_filename_list) { - nc_obs_initialize(); + nc_point_obs.init_buffer(); // Loop through the ASCII files, reading in the observations. At the end of // this loop, all of the observations will be in the _observations vector. @@ -156,15 +156,12 @@ bool FileHandler::writeNetcdfFile(const string &nc_filename) if (!_writeObservations()) return false; - // Add variable names - if (use_var_id) write_obs_var_names(obs_vars, obs_names); - // Close the netCDF file. _closeNetcdf(); - mlog << Debug(2) << "Finished processing " << obs_vars.obs_cnt - << " observations for " << obs_vars.hdr_cnt << " headers.\n"; + mlog << Debug(2) << "Finished processing " << nc_point_obs.get_obs_cnt() + << " observations for " << nc_point_obs.get_hdr_cnt() << " headers.\n"; return true; } @@ -249,16 +246,15 @@ bool FileHandler::_openNetcdf(const string &nc_filename) // // Define the NetCDF dimensions and variables // - init_nc_dims_vars_config(obs_vars, use_var_id); - obs_vars.attr_agl = true; + nc_point_obs.set_netcdf(_ncFile, true); + // Note: use_var_id was set by the handler + nc_point_obs.init_obs_vars(use_var_id, deflate_level, true); + nc_point_obs.set_nc_out_data(_observations, &summary_obs, _summaryInfo); - nc_out_data.processed_hdr_cnt = 0; - nc_out_data.deflate_level = deflate_level; - nc_out_data.observations = _observations; - nc_out_data.summary_obs = &summary_obs; - nc_out_data.summary_info = _summaryInfo; + int obs_cnt, hdr_cnt; + nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt); + nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, _programName); - init_netcdf_output(_ncFile, obs_vars, nc_out_data, _programName); // // Initialize the header and observation record counters @@ -270,48 +266,6 @@ bool FileHandler::_openNetcdf(const string &nc_filename) } -//////////////////////////////////////////////////////////////////////// - -//bool FileHandler::_writeHdrInfo(const ConcatString &hdr_typ, -// const ConcatString &hdr_sid, -// const time_t hdr_vld, -// double lat, double lon, double elv) { -// // -// // Increment header count before writing -// // -// _hdrNum++; -// write_nc_header(obs_vars, hdr_typ, hdr_sid, hdr_vld, lat, lon, elv); -// -// return true; -//} - -//////////////////////////////////////////////////////////////////////// - -//bool FileHandler::_writeObsInfo(int gc, float prs, float hgt, float obs, -// const ConcatString &qty) { -// float obs_arr[OBS_ARRAY_LEN]; -// ConcatString obs_qty; -// -// // -// // Increment observation count before writing -// // -// _obsNum++; -// -// // -// // Build the observation array -// // -// obs_arr[0] = _hdrNum; // Index of header -// obs_arr[1] = gc; // GRIB code corresponding to the observation type -// obs_arr[2] = prs; // Pressure level (hPa) or accumulation interval (sec) -// obs_arr[3] = hgt; // Height in meters above sea level or ground level (msl or agl) -// obs_arr[4] = obs; // Observation value -// -// obs_qty = (qty.length() == 0 ? na_str : qty.text()); -// write_nc_observation(obs_vars, nc_data_buffer, obs_arr, obs_qty.text()); -// -// return true; -//} - //////////////////////////////////////////////////////////////////////// bool FileHandler::_addObservations(const Observation &obs) @@ -383,14 +337,8 @@ bool FileHandler::_addObservations(const Observation &obs) bool FileHandler::_writeObservations() { - write_observations(_ncFile, obs_vars, nc_out_data); - - int var_count = (use_var_id ? obs_names.n_elements() : 0); - if (var_count > 0) { - int unit_count = 0; - create_nc_obs_name_vars (obs_vars, _ncFile, var_count, unit_count, deflate_level); - write_obs_var_names(obs_vars, obs_names); - } + StringArray descs, units; + nc_point_obs.write_to_netcdf(obs_names, units, descs); return true; } diff --git a/met/src/tools/other/ascii2nc/file_handler.h b/met/src/tools/other/ascii2nc/file_handler.h index 5a160e3068..1a50209513 100644 --- a/met/src/tools/other/ascii2nc/file_handler.h +++ b/met/src/tools/other/ascii2nc/file_handler.h @@ -36,6 +36,7 @@ using namespace netCDF; #include "vx_summary.h" #include "nc_obs_util.h" +#include "nc_point_obs_out.h" #include "nc_summary.h" //////////////////////////////////////////////////////////////////////// @@ -85,8 +86,7 @@ class FileHandler // Variables for writing output NetCDF file NcFile *_ncFile; - NetcdfObsVars obs_vars; - NcObsOutputData nc_out_data; + MetNcPointObsOut nc_point_obs; long _nhdr; diff --git a/met/src/tools/other/ioda2nc/Makefile.am b/met/src/tools/other/ioda2nc/Makefile.am index 83d74ba5d1..5942b7460c 100644 --- a/met/src/tools/other/ioda2nc/Makefile.am +++ b/met/src/tools/other/ioda2nc/Makefile.am @@ -18,7 +18,6 @@ ioda2nc_LDFLAGS = ${MET_LDFLAGS} ioda2nc_LDADD = -lvx_stat_out \ -lvx_statistics \ -lvx_shapedata \ - -lvx_gsl_prob \ -lvx_analysis_util \ -lvx_data2d_factory \ -lvx_data2d_nc_met \ @@ -26,22 +25,21 @@ ioda2nc_LDADD = -lvx_stat_out \ -lvx_data2d_nc_pinterp \ -lvx_data2d_nccf \ $(PYTHON_LIBS) \ - -lvx_statistics \ -lvx_data2d \ -lvx_nc_obs \ -lvx_nc_util \ -lvx_regrid \ -lvx_grid \ + -lvx_pb_util \ + -lvx_summary \ -lvx_config \ -lvx_gsl_prob \ - -lvx_pb_util \ - -lvx_cal \ + -lvx_color \ -lvx_util \ - $(PYTHON_LIBS) \ -lvx_math \ - -lvx_color \ + -lvx_cal \ -lvx_log \ - -lvx_summary \ + $(PYTHON_LIBS) \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas $(BLIB_NAME) \ $(FLIBS) diff --git a/met/src/tools/other/ioda2nc/ioda2nc.cc b/met/src/tools/other/ioda2nc/ioda2nc.cc index a9a08722b6..f822a1b005 100644 --- a/met/src/tools/other/ioda2nc/ioda2nc.cc +++ b/met/src/tools/other/ioda2nc/ioda2nc.cc @@ -46,6 +46,7 @@ using namespace std; #include "vx_summary.h" #include "nc_obs_util.h" +#include "nc_point_obs_out.h" #include "nc_summary.h" //////////////////////////////////////////////////////////////////////// @@ -76,7 +77,6 @@ static ConcatString ncfile; // Input configuration file static ConcatString config_file; static IODA2NCConfInfo conf_info; -static NcObsOutputData nc_out_data; static Grid mask_grid; static MaskPlane mask_area; @@ -105,7 +105,7 @@ static IntArray filtered_times; static bool do_summary; static bool save_summary_only = false; static SummaryObs *summary_obs; -static NetcdfObsVars obs_vars; +static MetNcPointObsOut nc_point_obs; //////////////////////////////////////////////////////////////////////// @@ -205,7 +205,7 @@ void initialize() { n_total_obs = 0; - nc_obs_initialize(); + nc_point_obs.init_buffer(); core_dims.clear(); core_dims.add("nvars"); @@ -323,7 +323,10 @@ void open_netcdf() { } // Define netCDF variables - init_nc_dims_vars_config(obs_vars); + int deflate_level = compress_level; + if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression(); + nc_point_obs.set_netcdf(f_out, true); + nc_point_obs.init_obs_vars(true, deflate_level); // Add global attributes write_netcdf_global(f_out, ncfile.text(), program_name); @@ -335,7 +338,7 @@ void open_netcdf() { void process_ioda_file(int i_pb) { int npbmsg, npbmsg_total; - int idx, i_msg, i_read, n_file_obs, i_ret, n_hdr_obs; + int idx, i_msg, i_read, n_file_obs, n_hdr_obs; int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly; int rej_elv, rej_nobs; double x, y; @@ -525,7 +528,7 @@ void process_ioda_file(int i_pb) { << "trouble getting datetime\n\n"; exit(1); } - + StringArray raw_var_names; if(do_all_vars || obs_var_names.n() == 0) raw_var_names = obs_value_vars; else raw_var_names = obs_var_names; @@ -562,7 +565,7 @@ void process_ioda_file(int i_pb) { } // Initialize counts - i_ret = n_file_obs = i_msg = 0; + n_file_obs = i_msg = 0; rej_typ = rej_sid = rej_vld = rej_grid = rej_poly = 0; rej_elv = rej_nobs = 0; @@ -591,7 +594,7 @@ void process_ioda_file(int i_pb) { for(int idx=0; idx 0) { if(bin_count > 0 && (i_read+1)%bin_count == 0) { @@ -768,7 +771,7 @@ void process_ioda_file(int i_pb) { } // Store the index to the header data - obs_arr[0] = (float) get_nc_hdr_cur_index(); + obs_arr[0] = (float) nc_point_obs.get_hdr_index(); n_hdr_obs = 0; for(idx=0; idx 0) { - add_nc_header_to_array(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut, - hdr_lat, hdr_lon, hdr_elv); + //nc_point_obs.add_header(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut, + // hdr_lat, hdr_lon, hdr_elv); i_msg++; } else { @@ -810,8 +813,7 @@ void process_ioda_file(int i_pb) { cout << log_message << "\n"; } - int obs_buf_index = get_nc_obs_buf_index(); - if(obs_buf_index > 0) write_nc_obs_buffer(obs_buf_index); + nc_point_obs.write_observation(); if(mlog.verbosity_level() > 0) cout << "\n" << flush; @@ -831,7 +833,7 @@ void process_ioda_file(int i_pb) { << rej_elv << "\n" << "Rejected based on zero observations\t= " << rej_nobs << "\n" - << "Total Records retained\t\t= " + << "Total Records retained\t\t\t= " << i_msg << "\n" << "Total observations retained or derived\t= " << n_file_obs << "\n"; @@ -898,22 +900,16 @@ void process_ioda_file(int i_pb) { //////////////////////////////////////////////////////////////////////// void write_netcdf_hdr_data() { + int obs_cnt, hdr_cnt; + const long hdr_count = (long) nc_point_obs.get_hdr_index(); static const string method_name = "\nwrite_netcdf_hdr_data()"; - const long hdr_count = (long) get_nc_hdr_cur_index(); - int deflate_level = compress_level; - if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression(); - - nc_out_data.processed_hdr_cnt = hdr_count; - nc_out_data.deflate_level = deflate_level; - nc_out_data.observations = observations; - nc_out_data.summary_obs = summary_obs; - nc_out_data.summary_info = conf_info.getSummaryInfo(); - - init_netcdf_output(f_out, obs_vars, nc_out_data, program_name); + nc_point_obs.set_nc_out_data(observations, summary_obs, conf_info.getSummaryInfo()); + nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt); + nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, program_name); // Check for no messages retained - if(obs_vars.hdr_cnt <= 0) { + if(hdr_cnt <= 0) { mlog << Error << method_name << " -> " << "No IODA reocrds retained. Nothing to write.\n\n"; // Delete the NetCDF file @@ -922,8 +918,6 @@ void write_netcdf_hdr_data() { } // Make sure all obs data is processed before handling header - write_observations(f_out, obs_vars, nc_out_data); - StringArray nc_var_name_arr; StringArray nc_var_unit_arr; StringArray nc_var_desc_arr; @@ -940,10 +934,7 @@ void write_netcdf_hdr_data() { nc_var_desc_arr.add(obs_var_descs[i]); } - create_nc_obs_name_vars(obs_vars, f_out, var_count, units_count, deflate_level); - write_obs_var_names(obs_vars, nc_var_name_arr); - write_obs_var_units(obs_vars, nc_var_unit_arr); - write_obs_var_descriptions(obs_vars, nc_var_desc_arr); + nc_point_obs.write_to_netcdf(nc_var_name_arr, nc_var_unit_arr, nc_var_desc_arr); return; } @@ -981,6 +972,8 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ, void clean_up() { + nc_point_obs.close(); + if(f_out) { delete f_out; f_out = (NcFile *) 0; diff --git a/met/src/tools/other/lidar2nc/Makefile.am b/met/src/tools/other/lidar2nc/Makefile.am index 118ea7efd4..9563167eee 100644 --- a/met/src/tools/other/lidar2nc/Makefile.am +++ b/met/src/tools/other/lidar2nc/Makefile.am @@ -45,8 +45,11 @@ lidar2nc_LDADD = -lvx_shapedata \ -lvx_grid \ -lvx_util \ -lvx_config \ + -lvx_gsl_prob \ -lvx_math \ -lvx_cal \ -lvx_log \ + $(PYTHON_LIBS) \ + -lvx_summary \ -lmfhdf -ldf -ljpeg \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lz diff --git a/met/src/tools/other/lidar2nc/lidar2nc.cc b/met/src/tools/other/lidar2nc/lidar2nc.cc index 627c1f70c6..547692cf24 100644 --- a/met/src/tools/other/lidar2nc/lidar2nc.cc +++ b/met/src/tools/other/lidar2nc/lidar2nc.cc @@ -60,20 +60,21 @@ using namespace std; #include "calipso_5km.h" #include "nc_obs_util.h" +#include "nc_point_obs_out.h" //////////////////////////////////////////////////////////////////////// -struct NcHeaderData *header_data = get_hdr_data_buffer(); - static ConcatString program_name; static CommandLine cline; static const int na_len = strlen(na_str); -static NetcdfObsVars obs_vars; static IntArray valid_times; +static NcFile *ncf; +static MetNcPointObsOut nc_point_obs; + //////////////////////////////////////////////////////////////////////// @@ -96,7 +97,7 @@ static void set_compress (const StringArray &); static void process_calipso_file (NcFile *, const char * filename); -static void write_nc_record(NetcdfObsVars a_obs_vars, const float * f, int qc_value = -1); +static void write_nc_record(const float * f, int qc_value = -1); //////////////////////////////////////////////////////////////////////// @@ -136,7 +137,8 @@ if ( output_filename.empty() ) usage(); // open the output file // -static NcFile *ncf = open_ncfile(output_filename.text(), true); +ncf = open_ncfile(output_filename.text(), true); +nc_point_obs.set_netcdf(ncf); // // process the lidar file @@ -146,6 +148,7 @@ mlog << Debug(1) << "Processing Lidar File: " << cline[0] << "\n"; process_calipso_file(ncf, cline[0].c_str()); +nc_point_obs.close(); // // done @@ -261,7 +264,6 @@ return ( t ); //////////////////////////////////////////////////////////////////////// - void process_calipso_file(NcFile * out, const char * filename) { @@ -271,7 +273,7 @@ int hdf_sd_id; int n_data; NcDim nc_dim; NcVar nc_lat_var, nc_lon_var, nc_data_var, nc_time_var; - +const char *method_name = "process_calipso_file() "; // // open hdf file @@ -317,7 +319,7 @@ buf_size = max(buf_size, hdr_vld_bytes); buf_size = max(buf_size, hdr_arr_bytes); -mlog << Debug(1) << "Writing MET File:\t" << output_filename << "\n"; +mlog << Debug(1) << method_name << "Writing MET File:\t" << output_filename << "\n"; // @@ -330,23 +332,18 @@ mlog << Debug(1) << "Writing MET File:\t" << output_filename << "\n"; // // add the variables to the netcdf file // - + int obs_cnt = 0; bool use_var_id = false; - init_nc_dims_vars_config(obs_vars, use_var_id); - obs_vars.attr_agl = true; - create_nc_hdr_vars(obs_vars, out, n_data, deflate_level); - create_nc_obs_vars(obs_vars, out, deflate_level, use_var_id); - - if (!IS_INVALID_NC(obs_vars.strl2_dim)) { - NcDim str_dim; - string dim_name = GET_NC_NAME(obs_vars.strl2_dim); - str_dim = get_nc_dim(&obs_vars.hdr_typ_var, dim_name); - } + NetcdfObsVars *obs_vars = nc_point_obs.get_obs_vars(); + nc_point_obs.init_buffer(); + nc_point_obs.init_obs_vars(use_var_id, deflate_level, true); + //nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt); + nc_point_obs.init_netcdf(obs_cnt, nhdr_dim_size, program_name); + mlog << Debug(4) << method_name << " hdr_dim: " << nhdr_dim_size << "\n"; // // global attributes for netcdf output file // - const unixtime now = time(0); int month, day, year, hour, minute, second; ConcatString s; @@ -385,8 +382,7 @@ mlog << Debug(2) << "Processing Lidar points\t= " << n_data << "\n"; memset(ibuf, 0, n_data*sizeof(int)); -header_data->typ_array.add(hdr_typ_string); -obs_vars.hdr_typ_var.putVar(ibuf); +obs_vars->hdr_typ_var.putVar(ibuf); // // populate the hdr_sid variable @@ -394,8 +390,8 @@ obs_vars.hdr_typ_var.putVar(ibuf); memset(ibuf, 0, n_data*sizeof(int)); -header_data->sid_array.add(na_str); -obs_vars.hdr_sid_var.putVar(ibuf); +obs_vars->hdr_sid_var.putVar(ibuf); +nc_point_obs.add_header_strings(hdr_typ_string, na_str); // // populate the obs_qty variable @@ -435,9 +431,9 @@ for (j=0; jhdr_lat_var.putVar(fhdr_lat_buf); +obs_vars->hdr_lon_var.putVar(fhdr_lon_buf); +obs_vars->hdr_elv_var.putVar(fhdr_elv_buf); delete [] fhdr_lat_buf; delete [] fhdr_lon_buf; @@ -473,14 +469,15 @@ for (j=0; jvld_array.add(junk); + //header_data.vld_array.add(junk); + nc_point_obs.add_header_vld(junk); } ibuf[j] = v_idx; } // for j -obs_vars.hdr_vld_var.putVar(ibuf); +obs_vars->hdr_vld_var.putVar(ibuf); delete[] ibuf; @@ -502,62 +499,60 @@ for (j=0; j obs_vector; static vector< ConcatString > md_files; @@ -144,7 +145,7 @@ int main(int argc, char *argv[]) { // process_command_line(argc, argv); - nc_obs_initialize(); + nc_point_obs.init_buffer(); // // Process the MADIS file @@ -155,9 +156,7 @@ int main(int argc, char *argv[]) { process_madis_file((*it_mdfile).c_str()); } - bool use_var_id = true; - bool do_header = false; - int nhdr = get_nc_hdr_cur_index(); + int nhdr = nc_point_obs.get_obs_index(); if (conf_info.getSummaryInfo().flag) { int summmary_hdr_cnt = 0; @@ -172,7 +171,8 @@ int main(int argc, char *argv[]) { } setup_netcdf_out(nhdr); - write_observations(f_out, obs_vars, nc_out_data); + StringArray obs_names, descs, units; + nc_point_obs.write_to_netcdf(obs_names, units, descs); // // Deallocate memory and clean up @@ -354,6 +354,8 @@ void clean_up() { if (summary_obs) delete summary_obs; + nc_point_obs.close(); + // // Close the output NetCDF file // @@ -387,19 +389,22 @@ void setup_netcdf_out(int nhdr) { } bool use_var_id = false; - init_nc_dims_vars_config(obs_vars, use_var_id); - obs_vars.obs_cnt = obs_vector.size(); - mlog << Debug(5) << "setup_netcdf_out() nhdr:\t" << nhdr - << "\tobs_cnt:\t" << obs_vars.obs_cnt << "\n"; + int obs_cnt, hdr_cnt; + nc_point_obs.set_netcdf(f_out, true); + nc_point_obs.set_using_var_id(use_var_id); - nc_out_data.processed_hdr_cnt = 0; - nc_out_data.deflate_level = compress_level; - nc_out_data.observations = obs_vector; - nc_out_data.summary_obs = summary_obs; - nc_out_data.summary_info = conf_info.getSummaryInfo(); + NetcdfObsVars *obs_vars = nc_point_obs.get_obs_vars(); + obs_vars->deflate_level = compress_level; - init_netcdf_output(f_out, obs_vars, nc_out_data, program_name); + //obs_vars.reset(use_var_id); + obs_vars->obs_cnt = obs_vector.size(); + mlog << Debug(5) << "setup_netcdf_out() nhdr:\t" << nhdr + << "\tobs_cnt:\t" << obs_vars->obs_cnt << "\n"; + nc_point_obs.set_nc_out_data(obs_vector, summary_obs, conf_info.getSummaryInfo()); + nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt); + nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, program_name); + // // Add the command line arguments that were applied. // @@ -744,7 +749,7 @@ void process_madis_metar(NcFile *&f_in) { double tmp_dbl; time_t hdr_vld; ConcatString hdr_typ, hdr_sid; - float hdr_arr[hdr_arr_len], obs_arr[obs_arr_len], conversion; + float hdr_arr[HDR_ARRAY_LEN], obs_arr[OBS_ARRAY_LEN], conversion; float wdir, wind, ugrd, vgrd; int count; StringArray missing_vars, missing_qty_vars; @@ -1140,18 +1145,16 @@ void process_madis_metar(NcFile *&f_in) { // Snow Cover obs_arr[2] = bad_data_float; obs_arr[4] = snowCover[i_idx]; - //count += process_obs(66, conversion, obs_arr, snowCoverQty[i_idx], - // snowCover_var, hdr_typ, hdr_sid, hdr_vld, - // hdr_arr[0], hdr_arr[1], hdr_arr[2]); - process_obs(66, conversion, obs_arr, snowCoverQty[i_idx], - snowCover_var, hdr_typ, hdr_sid, hdr_vld, - hdr_arr[0], hdr_arr[1], hdr_arr[2]); + count += process_obs(66, conversion, obs_arr, snowCoverQty[i_idx], + snowCover_var, hdr_typ, hdr_sid, hdr_vld, + hdr_arr[0], hdr_arr[1], hdr_arr[2]); } } // end for i_hdr print_rej_counts(); + mlog << Debug(5) << " Added " << count << "data\n"; // // Cleanup @@ -1172,7 +1175,7 @@ void process_madis_raob(NcFile *&f_in) { char qty; time_t hdr_vld; ConcatString hdr_typ, hdr_sid; - float hdr_arr[hdr_arr_len], obs_arr[obs_arr_len], conversion; + float hdr_arr[HDR_ARRAY_LEN], obs_arr[OBS_ARRAY_LEN], conversion; float wdir, wind, ugrd, vgrd; int count; StringArray missing_vars, missing_qty_vars; @@ -1594,7 +1597,7 @@ void process_madis_raob(NcFile *&f_in) { hdr_vld = (time_t)tmp_dbl; - hdr_idx = get_nc_hdr_cur_index(); + hdr_idx = nc_point_obs.get_obs_index(); // // Process the station name. @@ -1985,7 +1988,7 @@ void process_madis_profiler(NcFile *&f_in) { double tmp_dbl; time_t hdr_vld; ConcatString hdr_typ, hdr_sid; - float hdr_arr[hdr_arr_len], obs_arr[obs_arr_len], conversion; + float hdr_arr[HDR_ARRAY_LEN], obs_arr[OBS_ARRAY_LEN], conversion; float pressure; int count; StringArray missing_vars, missing_qty_vars; @@ -2170,7 +2173,7 @@ void process_madis_profiler(NcFile *&f_in) { hdr_vld = (time_t)tmp_dbl; - hdr_idx = get_nc_hdr_cur_index(); + hdr_idx = nc_point_obs.get_obs_index(); // // Initialize the observation array: hdr_id @@ -2243,7 +2246,7 @@ void process_madis_maritime(NcFile *&f_in) { double tmp_dbl; time_t hdr_vld; ConcatString hdr_typ, hdr_sid; - float hdr_arr[hdr_arr_len], obs_arr[obs_arr_len], conversion; + float hdr_arr[HDR_ARRAY_LEN], obs_arr[OBS_ARRAY_LEN], conversion; float pressure; int count; StringArray missing_vars, missing_qty_vars; @@ -2504,7 +2507,7 @@ void process_madis_maritime(NcFile *&f_in) { hdr_vld = (time_t)tmp_dbl; - hdr_idx = get_nc_hdr_cur_index(); + hdr_idx = nc_point_obs.get_obs_index(); // @@ -2595,18 +2598,16 @@ void process_madis_maritime(NcFile *&f_in) { // APCP_24 obs_arr[2] = 86400; obs_arr[4] = precip24Hour_arr[i_idx]; - //count += process_obs(61, conversion, obs_arr, precip24HourQty_arr[i_idx], - // in_precip24Hour_var, hdr_typ, hdr_sid, hdr_vld, - // hdr_arr[0], hdr_arr[1], hdr_arr[2]); - process_obs(61, conversion, obs_arr, precip24HourQty_arr[i_idx], - in_precip24Hour_var, hdr_typ, hdr_sid, hdr_vld, - hdr_arr[0], hdr_arr[1], hdr_arr[2]); + count += process_obs(61, conversion, obs_arr, precip24HourQty_arr[i_idx], + in_precip24Hour_var, hdr_typ, hdr_sid, hdr_vld, + hdr_arr[0], hdr_arr[1], hdr_arr[2]); } } // end for i_hdr print_rej_counts(); + mlog << Debug(5) << " Added " << count << "data\n"; // // Cleanup @@ -2625,7 +2626,7 @@ void process_madis_mesonet(NcFile *&f_in) { double tmp_dbl; time_t hdr_vld; ConcatString hdr_typ, hdr_sid; - float hdr_arr[hdr_arr_len], obs_arr[obs_arr_len], conversion; + float hdr_arr[HDR_ARRAY_LEN], obs_arr[OBS_ARRAY_LEN], conversion; float wdir, wind, ugrd, vgrd; int count; StringArray missing_vars, missing_qty_vars; @@ -2981,7 +2982,7 @@ void process_madis_mesonet(NcFile *&f_in) { if(is_bad_data(tmp_dbl)) continue; hdr_vld = (time_t)tmp_dbl; - hdr_idx = get_nc_hdr_cur_index(); + hdr_idx = nc_point_obs.get_obs_index(); // // Initialize the observation array: hdr_id, gc, lvl, hgt, ob @@ -3204,7 +3205,7 @@ void process_madis_acarsProfiles(NcFile *&f_in) { char qty; time_t hdr_vld; ConcatString hdr_typ, hdr_sid; - float hdr_arr[hdr_arr_len], obs_arr[obs_arr_len], conversion; + float hdr_arr[HDR_ARRAY_LEN], obs_arr[OBS_ARRAY_LEN], conversion; float pressure, wdir, wind, ugrd, vgrd; int count; StringArray missing_vars, missing_qty_vars; @@ -3428,7 +3429,7 @@ void process_madis_acarsProfiles(NcFile *&f_in) { i_cnt++; mlog << Debug(3) << " Mandatory Level: " << i_lvl << "\n"; - hdr_idx = get_nc_hdr_cur_index(); + hdr_idx = nc_point_obs.get_obs_index(); // // Use cur to index into the NetCDF variables. diff --git a/met/src/tools/other/madis2nc/madis2nc.h b/met/src/tools/other/madis2nc/madis2nc.h index 142698193e..74c03ed917 100644 --- a/met/src/tools/other/madis2nc/madis2nc.h +++ b/met/src/tools/other/madis2nc/madis2nc.h @@ -76,9 +76,6 @@ static const char *DEFAULT_CONFIG_FILENAME = "MET_BASE/config/Madis2NcConfig_default"; static const float fill_value = -9999.f; -static const int strl_len = 16; // Length of "YYYYMMDD_HHMMSS" -static const int hdr_arr_len = 3; // Observation header length -static const int obs_arr_len = 5; // Observation values length //////////////////////////////////////////////////////////////////////// // @@ -146,7 +143,6 @@ int processed_count; //////////////////////////////////////////////////////////////////////// static Madis2NcConfInfo conf_info; -static NcObsOutputData nc_out_data; static bool do_summary; static bool save_summary_only = false; diff --git a/met/src/tools/other/pb2nc/Makefile.am b/met/src/tools/other/pb2nc/Makefile.am index 6eb653a1d0..ce99c025a1 100644 --- a/met/src/tools/other/pb2nc/Makefile.am +++ b/met/src/tools/other/pb2nc/Makefile.am @@ -42,22 +42,21 @@ pb2nc_LDADD = -lvx_stat_out \ -lvx_data2d_nc_pinterp \ -lvx_data2d_nccf \ $(PYTHON_LIBS) \ - -lvx_statistics \ -lvx_data2d \ -lvx_nc_obs \ -lvx_nc_util \ -lvx_regrid \ -lvx_grid \ + -lvx_pb_util \ + -lvx_summary \ -lvx_config \ -lvx_gsl_prob \ - -lvx_pb_util \ - -lvx_cal \ + -lvx_color \ -lvx_util \ - $(PYTHON_LIBS) \ -lvx_math \ - -lvx_color \ + -lvx_cal \ -lvx_log \ - -lvx_summary \ + $(PYTHON_LIBS) \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas $(BLIB_NAME) \ $(FLIBS) diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index 1113290c76..a374bfc6f0 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -88,6 +88,7 @@ using namespace std; #include "vx_summary.h" #include "nc_obs_util.h" +#include "nc_point_obs_out.h" #include "nc_summary.h" //////////////////////////////////////////////////////////////////////// @@ -130,11 +131,6 @@ static const int mxr8nm = 8; // Maximum number of BUFR variable types // Length of the "YYYYMMDD_HHMMSS" string -static const int strl_len = 16; -// Observation header length -static const int hdr_arr_len = 3; -// Observation values length -static const int obs_arr_len = 5; static const int COUNT_THRESHOLD = 5; // File unit number for opening the PrepBufr file @@ -204,7 +200,7 @@ static ConcatString ncfile; // Input configuration file static ConcatString config_file; static PB2NCConfInfo conf_info; -static NcObsOutputData nc_out_data; +static MetNcPointObsOut nc_point_obs; // Beginning and ending retention times static unixtime valid_beg_ut, valid_end_ut; @@ -217,7 +213,6 @@ static int nmsg_percent = -1; static bool dump_flag = false; static ConcatString dump_dir = (string)"."; -static bool obs_to_vector = true; static bool do_all_vars = false; static bool override_vars = false; static bool collect_metadata = false; @@ -295,8 +290,6 @@ static map variableTypeMap; static bool do_summary; static SummaryObs *summary_obs; -static NetcdfObsVars obs_vars; - //////////////////////////////////////////////////////////////////////// @@ -471,7 +464,7 @@ void initialize() { n_total_obs = 0; - nc_obs_initialize(); + nc_point_obs.init_buffer(); prepbufr_vars.clear(); prepbufr_hdrs.clear(); @@ -784,18 +777,11 @@ void open_netcdf() { } // Define netCDF variables - init_nc_dims_vars_config(obs_vars); - obs_vars.attr_pb2nc = true; - - if (!obs_to_vector) { - - create_nc_obs_vars(obs_vars, f_out, - (compress_level >= 0 ? compress_level : - conf_info.conf.nc_compression())); - - // Add global attributes - write_netcdf_global(f_out, ncfile.text(), program_name); - } + bool use_var_id = true; + int deflate_level = compress_level; + if (deflate_level < 0) deflate_level = conf_info.conf.nc_compression(); + nc_point_obs.set_netcdf(f_out, true); + nc_point_obs.init_obs_vars(use_var_id, deflate_level); return; } @@ -835,7 +821,7 @@ void process_pbfile(int i_pb) { unixtime hdr_vld_ut = (unixtime) 0; float quality_mark, dl_category; - float obs_arr[obs_arr_len]; + float obs_arr[OBS_ARRAY_LEN]; float pqtzuv[mxr8vt], pqtzuv_qty[mxr8vt]; const int debug_level_for_performance = 3; @@ -1024,7 +1010,7 @@ void process_pbfile(int i_pb) { diff_file_time_count = 0; cycle_minute = missing_cycle_minute; // initialize - for (int idx=0; idx mxr8lv) { @@ -1757,7 +1743,7 @@ void process_pbfile(int i_pb) { } mlog << Debug(10) << "var: " << var_name << " nlev2: " << nlev2 << ", vIdx: " << vIdx << ", obs_data_idx: " - << get_nc_obs_buf_index() << ", nlev: " << nlev << "\n"; + << nc_point_obs.get_obs_index() << ", nlev: " << nlev << "\n"; // Search through the vertical levels for(lv=0; lv 0) { - add_nc_header_to_array(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut, - hdr_lat, hdr_lon, hdr_elv); + nc_point_obs.add_header(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut, + hdr_lat, hdr_lon, hdr_elv); if (is_prepbufr) { - add_nc_header_prepbufr(pb_report_type, in_report_type, instrument_type); + nc_point_obs.add_header_prepbufr(pb_report_type, in_report_type, instrument_type); } i_msg++; @@ -1911,8 +1897,7 @@ void process_pbfile(int i_pb) { } mlog << Warning << "\n"; } - int obs_buf_index = get_nc_obs_buf_index(); - if (obs_buf_index > 0) write_nc_obs_buffer(obs_buf_index); + nc_point_obs.write_observation(); if(mlog.verbosity_level() > 0) cout << "\n" << flush; @@ -2399,80 +2384,37 @@ void process_pbfile_metadata(int i_pb) { void write_netcdf_hdr_data() { long dim_count, pb_hdr_count; bool is_prepbufr = is_prepbufr_file(&event_names); - static const string method_name = "\nwrite_netcdf_hdr_data()"; + TimeSummaryInfo summary_info = conf_info.getSummaryInfo(); + static const string method_name = "write_netcdf_hdr_data() "; - pb_hdr_count = (long) get_nc_hdr_cur_index(); - if (obs_to_vector) { - int deflate_level = compress_level; - if (deflate_level < 0) deflate_level = conf_info.conf.nc_compression(); + pb_hdr_count = (long) nc_point_obs.get_hdr_index(); + nc_point_obs.set_nc_out_data(observations, summary_obs, + summary_info, pb_hdr_count); - nc_out_data.processed_hdr_cnt = pb_hdr_count; - nc_out_data.deflate_level = deflate_level; - nc_out_data.observations = observations; - nc_out_data.summary_obs = summary_obs; - nc_out_data.summary_info = conf_info.getSummaryInfo(); - - init_netcdf_output(f_out, obs_vars, nc_out_data, program_name); - dim_count = obs_vars.hdr_cnt; - } - else { - dim_count = pb_hdr_count; - if (do_summary) { - int summary_hdr_cnt = summary_obs->countSummaryHeaders(); - if (save_summary_only) - dim_count = summary_hdr_cnt; - else - dim_count += summary_hdr_cnt; - } - } + int obs_cnt, hdr_cnt; + nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt); + dim_count = hdr_cnt; // Check for no messages retained if(dim_count <= 0) { - mlog << Error << "\n" << method_name << " -> " + mlog << Error << "\n" << method_name << "-> " << "No PrepBufr messages retained. Nothing to write.\n\n"; // Delete the NetCDF file remove_temp_file(ncfile); exit(1); } - int deflate_level = compress_level; - if (deflate_level < 0) deflate_level = conf_info.conf.nc_compression(); + nc_point_obs.get_obs_vars()->attr_pb2nc = true; + nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, program_name); + if (is_prepbufr) { - if (!nc_out_data.summary_info.flag || nc_out_data.summary_info.raw_data) - create_nc_pb_hdrs(obs_vars, f_out, pb_hdr_count, deflate_level); + if (!summary_info.flag || summary_info.raw_data) { + nc_point_obs.create_pb_hdrs(pb_hdr_count); + } } // Make sure all obs data is processed before handling header - if (obs_to_vector) { - write_observations(f_out, obs_vars, nc_out_data); - } - else { - if (do_summary) { - // Write out the summary data - if (save_summary_only) reset_header_buffer(pb_hdr_count, true); - write_nc_observations(obs_vars, summary_obs->getSummaries()); - mlog << Debug(4) << "write_netcdf_hdr_data obs count: " - << (int)summary_obs->getObservations().size() - << " summary count: " << (int)summary_obs->getSummaries().size() - << " header count: " << dim_count - << " summary header count: " << (dim_count-pb_hdr_count) << "\n"; - - TimeSummaryInfo summaryInfo = conf_info.getSummaryInfo(); - if (summaryInfo.flag) { - write_summary_attributes(f_out, summaryInfo); - } - } - - create_nc_hdr_vars(obs_vars, f_out, dim_count, deflate_level); - create_nc_table_vars(obs_vars, f_out, deflate_level); - - // Write out the header data - write_nc_arr_headers(obs_vars); - if (get_nc_hdr_cur_index() > 0) { - // Write out the remaining header data - write_nc_buf_headers(obs_vars); - } - } + nc_point_obs.write_observation(); StringArray nc_var_name_arr; StringArray nc_var_unit_arr; @@ -2512,11 +2454,9 @@ void write_netcdf_hdr_data() { } // end for i - dim_count = bufr_obs_name_arr.n_elements(); - create_nc_obs_name_vars (obs_vars, f_out, dim_count, dim_count, deflate_level); - write_obs_var_names (obs_vars, nc_var_name_arr); - write_obs_var_units (obs_vars, nc_var_unit_arr); - write_obs_var_descriptions (obs_vars, nc_var_desc_arr); + bool using_obs_vars = true; + nc_point_obs.write_to_netcdf(nc_var_name_arr, nc_var_unit_arr, + nc_var_desc_arr); return; } @@ -2538,38 +2478,20 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ, obs_qty.format("%d", quality_code); } - if (obs_to_vector) { - int var_index = obs_arr[1]; - //assert(var_index >= 0 && strlen("Variable index can't be negative")); - string var_name = bufr_obs_name_arr[var_index]; - Observation obs = Observation(hdr_typ.text(), - hdr_sid.text(), - hdr_vld, - hdr_lat, hdr_lon, hdr_elv, - obs_qty.text(), - var_index, - obs_arr[2], obs_arr[3], obs_arr[4], - var_name); - obs.setHeaderIndex(obs_arr[0]); - observations.push_back(obs); - if (do_summary) summary_obs->addObservationObj(obs); - } - else { - if (!save_summary_only) - write_nc_observation(obs_vars, obs_arr, obs_qty.text()); - if (do_summary) { - string var_name = bufr_obs_name_arr[(obs_arr[1])]; - conf_info.getSummaryInfo(); - summary_obs->addObservation( - hdr_typ.text(), - hdr_sid.text(), - hdr_vld, - hdr_lat, hdr_lon, hdr_elv, - obs_qty.text(), - obs_arr[1], obs_arr[2], obs_arr[3], obs_arr[4], - var_name); - } - } + int var_index = obs_arr[1]; + //assert(var_index >= 0 && strlen("Variable index can't be negative")); + string var_name = bufr_obs_name_arr[var_index]; + Observation obs = Observation(hdr_typ.text(), + hdr_sid.text(), + hdr_vld, + hdr_lat, hdr_lon, hdr_elv, + obs_qty.text(), + var_index, + obs_arr[2], obs_arr[3], obs_arr[4], + var_name); + obs.setHeaderIndex(obs_arr[0]); + observations.push_back(obs); + if (do_summary) summary_obs->addObservationObj(obs); return; } @@ -2577,6 +2499,8 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ, void clean_up() { + nc_point_obs.close(); + if(f_out) { delete f_out; f_out = (NcFile *) 0; @@ -3291,7 +3215,7 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu void merge_records(float *first_pqtzuv, map pqtzuv_map_pivot, map pqtzuv_map_aux, map &pqtzuv_map_merged) { - float cur_pres, next_pres; + float cur_pres; float *cur_pqtzuv, *next_pqtzuv, *prev_pqtzuv; float *pqtzuv_merged; std::map::iterator it_pivot, it_aux; @@ -3311,7 +3235,6 @@ void merge_records(float *first_pqtzuv, map pqtzuv_map_pivot, for (; it_aux!=pqtzuv_map_aux.end(); ++it_aux) { // Skip the records below the first mathcing/interpolated level if (it_pivot->first < it_aux->first) { - next_pres = it_aux->first; break; } prev_pqtzuv = (float *)it_aux->second; diff --git a/met/src/tools/other/plot_point_obs/Makefile.am b/met/src/tools/other/plot_point_obs/Makefile.am index 77e7c1711d..fffb055719 100644 --- a/met/src/tools/other/plot_point_obs/Makefile.am +++ b/met/src/tools/other/plot_point_obs/Makefile.am @@ -21,29 +21,29 @@ plot_point_obs_LDADD = -lvx_statistics \ -lvx_data2d_grib $(GRIB2_LIBS) \ -lvx_data2d_nc_pinterp \ -lvx_data2d_nccf \ - -lvx_nc_util \ + $(PYTHON_LIBS) \ -lvx_data2d \ - -lvx_nc_obs \ - -lvx_nc_util \ + -lvx_nc_util \ + -lvx_statistics \ + -lvx_nc_obs \ + -lvx_nc_util \ -lvx_gnomon \ -lvx_regrid \ -lvx_plot_util \ -lvx_render \ -lvx_pxm \ - -lvx_color \ -lvx_ps \ -lvx_afm \ -lvx_nav \ -lvx_grid \ -lvx_config \ -lvx_gsl_prob \ - -lvx_cal \ + -lvx_color \ -lvx_util \ - $(PYTHON_LIBS) \ - -lvx_statistics \ -lvx_math \ - -lvx_color \ + -lvx_cal \ -lvx_log \ + $(PYTHON_LIBS) \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas EXTRA_DIST = plot_point_obs.h \ plot_point_obs_conf_info.h diff --git a/met/src/tools/other/plot_point_obs/plot_point_obs.cc b/met/src/tools/other/plot_point_obs/plot_point_obs.cc index 6ffee6c271..1ba1b9938b 100644 --- a/met/src/tools/other/plot_point_obs/plot_point_obs.cc +++ b/met/src/tools/other/plot_point_obs/plot_point_obs.cc @@ -65,6 +65,7 @@ using namespace std; #include "vx_render.h" #include "vx_plot_util.h" #include "nc_obs_util.h" +#include "nc_point_obs_in.h" //////////////////////////////////////////////////////////////////////// @@ -147,272 +148,108 @@ int main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////////////// void process_point_obs(const char *point_obs_filename) { - int h, v; - int obs_hid_block[DEF_NC_BUFFER_SIZE]; - int obs_vid_block[DEF_NC_BUFFER_SIZE]; - int obs_qty_block[DEF_NC_BUFFER_SIZE]; - float obs_lvl_block[DEF_NC_BUFFER_SIZE]; - float obs_hgt_block[DEF_NC_BUFFER_SIZE]; - float obs_val_block[DEF_NC_BUFFER_SIZE]; - float obs_arr_block[DEF_NC_BUFFER_SIZE][OBS_ARRAY_LEN]; + int h, v, i_obs; + const char *method_name = "\nprocess_point_obs() -> "; + const char *method_name_s = "\nprocess_point_obs() "; // Open the netCDF point observation file mlog << Debug(1) << "Reading point observation file: " << point_obs_filename << "\n"; - NcFile * f_in = open_ncfile(point_obs_filename); + MetNcPointObsIn nc_point_obs; + if(!nc_point_obs.open(point_obs_filename)) { + nc_point_obs.close(); - if(!f_in || IS_INVALID_NC_P(f_in)) { - mlog << Error << "\nprocess_point_obs() -> " + mlog << Error << "\n" << method_name << "trouble opening point observation file " << point_obs_filename << ".\n\n"; - delete f_in; - f_in = (NcFile *) 0; exit(1); } // Read the dimensions and variables - NetcdfObsVars obsVars; - read_nc_dims_vars(obsVars, f_in); - bool use_var_id = obsVars.use_var_id; - bool use_qty_idx = IS_VALID_NC(obsVars.obs_qty_tbl_var); + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(point_obs_filename, method_name_s); // exit if missing dims/vars + nc_point_obs.read_obs_data(); + + bool use_var_id = nc_point_obs.is_using_var_id(); // Print warning about ineffective command line arguments if(use_var_id && ivar.n() != 0) { - mlog << Warning << "\n-gc option is ignored!\n\n"; + mlog << Warning << "\n" << method_name << "-gc option is ignored!\n\n"; } if(!use_var_id && svar.n() != 0) { - mlog << Warning << "\n-obs_var option is ignored!\n\n"; + mlog << Warning << "\n" << method_name << "-obs_var option is ignored!\n\n"; } - if(IS_INVALID_NC(obsVars.hdr_dim) || - IS_INVALID_NC(obsVars.obs_dim) || - IS_INVALID_NC(obsVars.strl_dim) || - IS_INVALID_NC(obsVars.hdr_typ_var) || - IS_INVALID_NC(obsVars.hdr_sid_var) || - IS_INVALID_NC(obsVars.hdr_vld_var)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble reading point observation file " - << point_obs_filename << ".\n\n"; - exit(1); - } - if(is_version_less_than_1_02(f_in)) { - if(IS_INVALID_NC(obsVars.hdr_arr_dim) || - IS_INVALID_NC(obsVars.obs_arr_dim) || - IS_INVALID_NC(obsVars.hdr_arr_var) || - IS_INVALID_NC(obsVars.obs_arr_var)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble reading point observation file " - << point_obs_filename << "(2D variables).\n\n"; - exit(1); - } - } - else { - if(IS_INVALID_NC(obsVars.hdr_lat_var) || - IS_INVALID_NC(obsVars.hdr_typ_tbl_var) || - IS_INVALID_NC(obsVars.obs_qty_tbl_var) || - IS_INVALID_NC(obsVars.obs_val_var)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble reading point observation file " - << point_obs_filename << "(header or obs).\n\n"; - exit(1); - } - } - - long nhdr_count = get_dim_size(&obsVars.hdr_dim); - long nobs_count = get_dim_size(&obsVars.obs_dim); + long nhdr_count = nc_point_obs.get_hdr_cnt(); + long nobs_count = nc_point_obs.get_obs_cnt(); mlog << Debug(2) << "Processing " << nobs_count << " observations at " << nhdr_count << " locations.\n"; // Get the corresponding header: // message type, staton_id, valid_time, and lat/lon/elv - NcHeaderData header_data = get_nc_hdr_data(obsVars); - bool use_hdr_arr = !IS_INVALID_NC(obsVars.hdr_arr_var); - bool use_obs_arr = !IS_INVALID_NC(obsVars.obs_arr_var); + bool use_obs_arr = nc_point_obs.is_using_obs_arr(); - int hdr_arr_len = use_hdr_arr ? get_dim_size(&obsVars.hdr_arr_dim) : - HDR_ARRAY_LEN; - int obs_arr_len = use_obs_arr ? get_dim_size(&obsVars.obs_arr_dim) : - OBS_ARRAY_LEN; - int buf_size; + int buf_size = (nobs_count > DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : nobs_count; // Allocate space to store the data - float hdr_arr[hdr_arr_len]; - float obs_arr[obs_arr_len]; - - long offsets[2] = { 0, 0 }; - long lengths[2] = { 1, 1 }; - long offsets_1D[1] = { 0 }; - long lengths_1D[1] = { 1 }; - - if(use_var_id) { - NcVar obs_var_var = get_nc_var(f_in, nc_var_obs_var); - long var_count = get_dim_size(&obs_var_var, 0); - long var_len = get_dim_size(&obs_var_var, 1); - char obs_var_str[var_count][var_len]; - - lengths[0] = var_count; - lengths[1] = var_len; - if(!get_nc_data(&obs_var_var, - (char *) &obs_var_str[0], - lengths, offsets)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting " << nc_var_obs_var << "\n\n"; - exit(1); - } - for(int index=0; index " - << "trouble getting " << nc_var_obs_qty_tbl << "\n\n"; - exit(1); - } - for(int index=0; index DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : (nobs_count-i_start); - offsets[0] = i_start; - lengths[0] = buf_size; - offsets_1D[0] = i_start; - lengths_1D[0] = buf_size; - if(use_obs_arr) { - lengths[1] = obs_arr_len; - - // Read the current observation message - if(!get_nc_data(&obsVars.obs_arr_var, - (float *) &obs_arr_block[0], - lengths, offsets)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_arr\n\n"; - exit(1); - } - - lengths[1] = qty_len; - if(!get_nc_data(&obsVars.obs_qty_var, - (char *) qty_str_block, - lengths, offsets)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_qty\n\n"; - exit(1); - } - qty_list.clear(); - for(int index=0; index " - << "trouble getting obs_hid\n\n"; - exit(1); - } - if(!get_nc_data((IS_INVALID_NC(obsVars.obs_vid_var) ? - &obsVars.obs_gc_var : &obsVars.obs_vid_var), - (int *)&obs_vid_block[0], lengths_1D, offsets)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_hid\n\n"; - exit(1); - } - if(!get_nc_data(&obsVars.obs_lvl_var, - (float *) &obs_lvl_block[0], - lengths_1D, offsets_1D)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_lvl\n\n"; - exit(1); - } - if(!get_nc_data(&obsVars.obs_hgt_var, - (float *) &obs_hgt_block[0], - lengths_1D, offsets_1D)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_hgt\n\n"; - exit(1); - } - if(!get_nc_data(&obsVars.obs_val_var, - (float *) &obs_val_block[0], - lengths_1D, offsets_1D)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_val\n\n"; - exit(1); - } - if (use_qty_idx) { - if(!get_nc_data(&obsVars.obs_qty_var, - (int *) obs_qty_block, - lengths_1D, offsets_1D)) { - mlog << Error << "\nprocess_point_obs() -> " - << "trouble getting obs_qty\n\n"; - exit(1); - } - } + if (!nc_point_obs.read_obs_data(buf_size, i_start, (float *)obs_arr_block, + obs_qty_block, (char *)0)) { + exit(1); } int typ_idx, sid_idx, vld_idx; for(int i_offset=0; i_offset= 1.0E10 && obs_arr[1] >= 1.0E10) break; + int qty_offset = use_obs_arr ? i_obs : obs_qty_block[i_offset]; + obs_qty_str = qty_list[qty_offset]; + // Get the header index and variable type for this observation. h = nint(obs_arr[0]); v = nint(obs_arr[1]); - typ_idx = (use_obs_arr ? h : header_data.typ_idx_array[h]); - sid_idx = (use_obs_arr ? h : header_data.sid_idx_array[h]); - vld_idx = (use_obs_arr ? h : header_data.vld_idx_array[h]); + // Read the corresponding header array for this observation + // - the corresponding header type, header Station ID, and valid time + nc_point_obs.get_header(h, hdr_arr, hdr_typ_str, hdr_sid_str, hdr_vld_str); // Store data in an observation object Observation cur_obs( - header_data.typ_array[typ_idx], // message type - header_data.sid_array[sid_idx], // station id + hdr_typ_str, // message type + hdr_sid_str, // station id timestring_to_time_t( // valid time - header_data.vld_array[vld_idx].c_str()), - header_data.lat_array[h], // latitude - header_data.lon_array[h], // longitude - header_data.elv_array[h], // elevation - (use_qty_idx ? // quality string - qty_list[obs_qty_block[i_offset]] : - qty_list[i_offset]), + hdr_vld_str.c_str()), + hdr_arr[0], // latitude + hdr_arr[1], // longitude + hdr_arr[2], // elevation + obs_qty_str.c_str(), // quality string (use_var_id ? bad_data_int : v), // grib code (double) obs_arr[2], // pressure (double) obs_arr[3], // height @@ -426,7 +263,7 @@ void process_point_obs(const char *point_obs_filename) { } // end for i_start // Clean up - if(f_in) { delete f_in; f_in = (NcFile *) 0; } + nc_point_obs.close(); return; } diff --git a/met/src/tools/other/point2grid/Makefile.am b/met/src/tools/other/point2grid/Makefile.am index ccd38b1ac5..28ec81099a 100644 --- a/met/src/tools/other/point2grid/Makefile.am +++ b/met/src/tools/other/point2grid/Makefile.am @@ -20,19 +20,20 @@ point2grid_LDADD = -lvx_statistics \ -lvx_data2d_nc_met \ -lvx_data2d_grib $(GRIB2_LIBS) \ -lvx_data2d_nc_pinterp \ - $(PYTHON_LIBS) \ -lvx_data2d_nccf \ - -lvx_statistics \ - -lvx_nc_util \ + $(PYTHON_LIBS) \ -lvx_data2d \ + -lvx_nc_util \ + -lvx_nc_obs \ -lvx_gnomon \ -lvx_regrid \ -lvx_grid \ -lvx_config \ -lvx_gsl_prob \ - -lvx_cal \ + -lvx_color \ -lvx_util \ -lvx_math \ - -lvx_color \ + -lvx_cal \ -lvx_log \ + $(PYTHON_LIBS) \ -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas diff --git a/met/src/tools/other/point2grid/point2grid.cc b/met/src/tools/other/point2grid/point2grid.cc index f81a0c6bea..6402fff121 100644 --- a/met/src/tools/other/point2grid/point2grid.cc +++ b/met/src/tools/other/point2grid/point2grid.cc @@ -38,6 +38,8 @@ using namespace std; #include "vx_regrid.h" #include "vx_util.h" #include "vx_statistics.h" +#include "nc_obs_util.h" +#include "nc_point_obs_in.h" #include "point2grid_conf_info.h" @@ -612,22 +614,11 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, NcVar var_obs_gc, var_obs_var; clock_t start_clock = clock(); - bool use_var_id = true; bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_file() -> "; - - if (!get_dim(nc_in, ConcatString(nc_dim_nhdr), nhdr, true)) { - mlog << Error << "\n" << method_name - << "can not find the header dimension\"" << nc_dim_nhdr << "\".\n\n"; - exit(1); - } - if (!get_dim(nc_in, ConcatString(nc_dim_nobs), nobs, true)) { - mlog << Error << "\n" << method_name - << "can not find the obs dimension\"" << nc_dim_nobs << "\".\n\n"; - exit(1); - } + static const char *method_name_s = "process_point_file()"; // Check for at least one configuration string if(FieldSA.n() < 1) { @@ -636,70 +627,39 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, usage(); } - int *obs_ids = new int[nobs]; // grib_code or var_id - int *obs_hids = new int[nobs]; + MetNcPointObsIn nc_point_obs; + nc_point_obs.set_netcdf(nc_in, true); + // Read the dimensions and variables + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(GET_NC_NAME_P(nc_in).c_str(), method_name_s); // exit if missing dims/vars + // Read all obs data to compute the cell mapping + nc_point_obs.read_obs_data(); + NcHeaderData header_data = nc_point_obs.get_header_data(); + NcPointObsData obs_data = nc_point_obs.get_point_obs_data(); + + nhdr = nc_point_obs.get_hdr_cnt(); + nobs = nc_point_obs.get_obs_cnt(); + bool empty_input = (nhdr == 0 && nobs == 0); + bool use_var_id = nc_point_obs.is_using_var_id(); + float *hdr_lats = new float[nhdr]; float *hdr_lons = new float[nhdr]; - float *obs_lvls = new float[nobs]; - float *obs_hgts = new float[nobs]; - float *obs_vals = new float[nobs]; - int *hdr_typ_ids = new int[nhdr]; - int *hdr_vld_ids = new int[nhdr]; - int *obs_qty_ids = new int[nobs]; IntArray var_index_array; IntArray valid_time_array; - StringArray qc_tables; - StringArray var_names; - StringArray hdr_types; - StringArray hdr_valid_times; + StringArray qc_tables = nc_point_obs.get_qty_data(); + StringArray var_names = nc_point_obs.get_var_names(); + StringArray hdr_valid_times = header_data.vld_array; + hdr_valid_times.sort(); + + nc_point_obs.get_lats(hdr_lats); + nc_point_obs.get_lons(hdr_lons); - if (!get_nc_data_string_array(nc_in, nc_var_hdr_typ_tbl, &hdr_types)) exit(1); // Check the message types - prepare_message_types(hdr_types); + prepare_message_types(header_data.typ_array); // Check and read obs_vid and obs_var if exists bool success_to_read = true; - bool empty_input = (nhdr == 0 && nobs == 0); - NcVar obs_vid_var = get_var(nc_in, nc_var_obs_vid); - if (IS_VALID_NC(obs_vid_var)) { - if (success_to_read) success_to_read = get_nc_data_int_array(nc_in, nc_var_obs_vid, obs_ids); - if (success_to_read) success_to_read = get_nc_data_string_array(nc_in, nc_var_obs_var, &var_names); - } - else { - use_var_id = false; - if (success_to_read) success_to_read = get_nc_data_int_array(nc_in, nc_var_obs_gc, obs_ids, false); - if (!success_to_read) { - mlog << Error << "\n" << method_name - << "\"" << InputFilename << "\" is very old format. Not supported\n\n"; - } - } - if (!empty_input) { - if (success_to_read) - success_to_read = get_nc_data_int_array(nc_in, nc_var_obs_hid, obs_hids); - if (success_to_read) - success_to_read = get_nc_data_int_array(nc_in, nc_var_hdr_vld, hdr_vld_ids); - if (success_to_read) - success_to_read = get_nc_data_int_array(nc_in, nc_var_hdr_typ, hdr_typ_ids); - if (success_to_read) - success_to_read = get_nc_data_int_array(nc_in, nc_var_obs_qty, obs_qty_ids); - if (success_to_read) - success_to_read = get_nc_data_float_array(nc_in, nc_var_hdr_lat, hdr_lats); - if (success_to_read) - success_to_read = get_nc_data_float_array(nc_in, nc_var_hdr_lon, hdr_lons); - if (success_to_read) - success_to_read = get_nc_data_float_array(nc_in, nc_var_obs_lvl, obs_lvls); - if (success_to_read) - success_to_read = get_nc_data_float_array(nc_in, nc_var_obs_hgt, obs_hgts); - if (success_to_read) - success_to_read = get_nc_data_float_array(nc_in, nc_var_obs_val, obs_vals); - if (success_to_read) - success_to_read = get_nc_data_string_array( - nc_in, nc_var_hdr_vld_tbl, &hdr_valid_times); - if (success_to_read) - success_to_read = get_nc_data_string_array( - nc_in, nc_var_obs_qty_tbl, &qc_tables); - } if (success_to_read) { bool has_qc_flags = (qc_flags.n() > 0); IntArray qc_idx_array = prepare_qc_array(qc_flags, qc_tables); @@ -772,7 +732,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, else { bool not_found_grib_code = true; for (idx=0; idx= 4) { @@ -1111,16 +1071,9 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, } } - delete [] obs_ids; - delete [] obs_hids; delete [] hdr_lats; delete [] hdr_lons; - delete [] obs_lvls; - delete [] obs_hgts; - delete [] obs_vals; - delete [] hdr_typ_ids; - delete [] hdr_vld_ids; - delete [] obs_qty_ids; + nc_point_obs.close(); mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; @@ -1166,14 +1119,12 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, usage(); } - bool is_2d_time = false; unixtime valid_time = bad_data_int; valid_beg_ut = valid_end_ut = conf_info.valid_time; NcVar time_var = get_nc_var_time(nc_in); if( IS_VALID_NC(time_var) ) { if( 1 < get_dim_count(&time_var) ) { - is_2d_time = true; double max_time = bad_data_double; skip_times = new bool[from_size]; valid_times = new double[from_size];