diff --git a/met/data/config/Madis2NcConfig_default b/met/data/config/Madis2NcConfig_default index 41cfb3a477..e0cea4ce9d 100644 --- a/met/data/config/Madis2NcConfig_default +++ b/met/data/config/Madis2NcConfig_default @@ -16,6 +16,9 @@ // when the obs name is given instead of grib_code // +// Removed following variables from 2017 +mesonet_opt_var = [ "precip3hr", "precip6hr", "precip12hr", "precip10min" ]; + time_summary = { flag = FALSE; raw_data = FALSE; diff --git a/met/docs/Users_Guide/reformat_point.rst b/met/docs/Users_Guide/reformat_point.rst index 999393b72d..262500cbc4 100644 --- a/met/docs/Users_Guide/reformat_point.rst +++ b/met/docs/Users_Guide/reformat_point.rst @@ -677,7 +677,7 @@ Optional arguments for madis2nc An example of the madis2nc calling sequence is shown below: .. code-block:: none - + madis2nc sample_madis_obs.nc \ sample_madis_obs_met.nc -log madis.log -v 3 @@ -704,6 +704,16 @@ _________________ The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. +_________________ + + +.. code-block:: none + + mesonet_opt_var = [ "precip3hr", "precip6hr", "precip12hr", "precip10min" ]; + + +This entry is an array of variables which are optional variables for the MESONET input data. Above variables are not available from sometime in 2016. + __________________ diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index a46559ca02..4a8894af70 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -854,6 +854,12 @@ static const char conf_key_nc_output [] = "nc_output"; static const char conf_key_txt_output [] = "txt_output"; static const char conf_key_do_polylines_flag [] = "do_polylines"; +// +// MADIS2NC specific parameter key names +// + +static const char conf_key_mesonet_opt_var[] = "mesonet_opt_var"; + // // PB2NC specific parameter key names // diff --git a/met/src/tools/other/madis2nc/madis2nc.cc b/met/src/tools/other/madis2nc/madis2nc.cc index b1192f566f..b1126e4ce9 100644 --- a/met/src/tools/other/madis2nc/madis2nc.cc +++ b/met/src/tools/other/madis2nc/madis2nc.cc @@ -84,11 +84,15 @@ static void clean_up(); static void setup_netcdf_out(int nhdr); static bool get_filtered_nc_data(NcVar var, float *data, const long dim, - const long cur, const char *var_name); + const long cur, const char *var_name, bool required=true); +static bool get_filtered_nc_data_opt(NcVar var, float *data, const long dim, + const long cur, const char *var_name, StringArray optional_vars); + static bool get_filtered_nc_data_2d(NcVar var, int *data, const long *dim, - const long *cur, const char *var_name, bool count_bad=false); + const long *cur, const char *var_name); static bool get_filtered_nc_data_2d(NcVar var, float *data, const long *dim, - const long *cur, const char *var_name, bool count_bad=false); + const long *cur, const char *var_name); +static void get_nc_qty_data(NcVar *nc_var, char *nc_data_arr, int buf_size, int offset); static void check_quality_control_flag(int &value, const char qty, const char *var_name); static void check_quality_control_flag(float &value, const char qty, const char *var_name); @@ -415,7 +419,7 @@ void setup_netcdf_out(int nhdr) { static bool get_filtered_nc_data(NcVar var, float *data, const long dim, const long cur, - const char *var_name) { + const char *var_name, bool required) { bool status = false; float in_fill_value; @@ -439,15 +443,15 @@ static bool get_filtered_nc_data(NcVar var, float *data, << "Fail to read data [" << var_name << "].\n\n"; } } - else { + else if (required) { mlog << Error << "\n" << method_name << "Can not read a NetCDF data because the variable [" << var_name << "] is missing.\n\n"; } if (!status) { for (int idx=0; idx 0; if (missing_vars.n() > 0) { - mlog << Error << "\n" << method_name << "Please check if the input is a MESONET.\n\n"; - for (int idx=0; idx0) { + mlog << Error << "\n" << method_name << "Please check if the input is a MESONET.\n\n"; + for (int idx=0; idx 0) { - for (int idx=0; idx BUFFER_SIZE) ? BUFFER_SIZE: (my_rec_end - i_hdr_s); float hdr_lat_arr[buf_size]; @@ -2859,76 +2865,54 @@ void process_madis_mesonet(NcFile *&f_in) { get_nc_data(&in_hdr_lon_var, hdr_lon_arr, buf_size, i_hdr_s); get_filtered_nc_data(in_hdr_elv_var, hdr_elv_arr, buf_size, i_hdr_s, "eleveation"); - if (IS_VALID_NC(in_temperatureQty_var)) get_nc_data(&in_temperatureQty_var, temperatureQty_arr, buf_size, i_hdr_s); - else memset(temperatureQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_dewpointQty_var)) get_nc_data(&in_dewpointQty_var, dewpointQty_arr, buf_size, i_hdr_s); - else memset(dewpointQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_relHumidityQty_var)) get_nc_data(&in_relHumidityQty_var, relHumidityQty_arr, buf_size, i_hdr_s); - else memset(relHumidityQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_stationPressureQty_var)) get_nc_data(&in_stationPressureQty_var, stationPressureQty_arr, buf_size, i_hdr_s); - else memset(stationPressureQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_seaLevelPressureQty_var)) get_nc_data(&in_seaLevelPressureQty_var, seaLevelPressureQty_arr, buf_size, i_hdr_s); - else memset(seaLevelPressureQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_windDirQty_var)) get_nc_data(&in_windDirQty_var, windDirQty_arr, buf_size, i_hdr_s); - else memset(windDirQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_windSpeedQty_var)) get_nc_data(&in_windSpeedQty_var, windSpeedQty_arr, buf_size, i_hdr_s); - else memset(windSpeedQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_windGustQty_var)) get_nc_data(&in_windGustQty_var, windGustQty_arr, buf_size, i_hdr_s); - else memset(windGustQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_visibilityQty_var)) get_nc_data(&in_visibilityQty_var, visibilityQty_arr, buf_size, i_hdr_s); - else memset(visibilityQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_precipRateQty_var)) get_nc_data(&in_precipRateQty_var, precipRateQty_arr, buf_size, i_hdr_s); - else memset(precipRateQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_solarRadiationQty_var)) get_nc_data(&in_solarRadiationQty_var, solarRadiationQty_arr, buf_size, i_hdr_s); - else memset(solarRadiationQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_seaSurfaceTempQty_var)) get_nc_data(&in_seaSurfaceTempQty_var, seaSurfaceTempQty_arr, buf_size, i_hdr_s); - else memset(seaSurfaceTempQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_totalColumnPWVQty_var)) get_nc_data(&in_totalColumnPWVQty_var, totalColumnPWVQty_arr, buf_size, i_hdr_s); - else memset(totalColumnPWVQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_soilTemperatureQty_var)) get_nc_data(&in_soilTemperatureQty_var, soilTemperatureQty_arr, buf_size, i_hdr_s); - else memset(soilTemperatureQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_minTemp24HourQty_var)) get_nc_data(&in_minTemp24HourQty_var, minTemp24HourQty_arr, buf_size, i_hdr_s); - else memset(minTemp24HourQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_maxTemp24HourQty_var)) get_nc_data(&in_maxTemp24HourQty_var, maxTemp24HourQty_arr, buf_size, i_hdr_s); - else memset(maxTemp24HourQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_precip3hrQty_var)) get_nc_data(&in_precip3hrQty_var, precip3hrQty_arr, buf_size, i_hdr_s); - else memset(precip3hrQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_precip6hrQty_var)) get_nc_data(&in_precip6hrQty_var, precip6hrQty_arr, buf_size, i_hdr_s); - else memset(precip6hrQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_precip12hrQty_var)) get_nc_data(&in_precip12hrQty_var, precip12hrQty_arr, buf_size, i_hdr_s); - else memset(precip12hrQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_precip10minQty_var)) get_nc_data(&in_precip10minQty_var, precip10minQty_arr, buf_size, i_hdr_s); - else memset(precip10minQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_precip1minQty_var)) get_nc_data(&in_precip1minQty_var, precip1minQty_arr, buf_size, i_hdr_s); - else memset(precip1minQty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_windDir10Qty_var)) get_nc_data(&in_windDir10Qty_var, windDir10Qty_arr, buf_size, i_hdr_s); - else memset(windDir10Qty_arr, 0, buf_size*sizeof(char)); - if (IS_VALID_NC(in_windSpeed10Qty_var)) get_nc_data(&in_windSpeed10Qty_var, windSpeed10Qty_arr, buf_size, i_hdr_s); - else memset(windSpeed10Qty_arr, 0, buf_size*sizeof(char)); - - get_filtered_nc_data(in_temperature_var, temperature_arr, buf_size, i_hdr_s, "temperature" ); - get_filtered_nc_data(in_dewpoint_var, dewpoint_arr, buf_size, i_hdr_s, "dewpoint" ); - get_filtered_nc_data(in_relHumidity_var, relHumidity_arr, buf_size, i_hdr_s, "relHumidity" ); - get_filtered_nc_data(in_stationPressure_var, stationPressure_arr, buf_size, i_hdr_s, "stationPressure" ); - get_filtered_nc_data(in_seaLevelPressure_var, seaLevelPressure_arr, buf_size, i_hdr_s, "seaLevelPressure"); - get_filtered_nc_data(in_windDir_var, windDir_arr, buf_size, i_hdr_s, "windDir" ); - get_filtered_nc_data(in_windSpeed_var, windSpeed_arr, buf_size, i_hdr_s, "windSpeed" ); - get_filtered_nc_data(in_windGust_var, windGust_arr, buf_size, i_hdr_s, "windGust" ); - get_filtered_nc_data(in_visibility_var, visibility_arr, buf_size, i_hdr_s, "visibility" ); - get_filtered_nc_data(in_precipRate_var, precipRate_arr, buf_size, i_hdr_s, "precipRate" ); - get_filtered_nc_data(in_solarRadiation_var, solarRadiation_arr, buf_size, i_hdr_s, "solarRadiation" ); - get_filtered_nc_data(in_seaSurfaceTemp_var, seaSurfaceTemp_arr, buf_size, i_hdr_s, "seaSurfaceTemp" ); - get_filtered_nc_data(in_totalColumnPWV_var, totalColumnPWV_arr, buf_size, i_hdr_s, "totalColumnPWV" ); - get_filtered_nc_data(in_soilTemperature_var, soilTemperature_arr, buf_size, i_hdr_s, "soilTemperature" ); - get_filtered_nc_data(in_minTemp24Hour_var, minTemp24Hour_arr, buf_size, i_hdr_s, "minTemp24Hour" ); - get_filtered_nc_data(in_maxTemp24Hour_var, maxTemp24Hour_arr, buf_size, i_hdr_s, "maxTemp24Hour" ); - get_filtered_nc_data(in_precip3hr_var, precip3hr_arr, buf_size, i_hdr_s, "precip3hr" ); - get_filtered_nc_data(in_precip6hr_var, precip6hr_arr, buf_size, i_hdr_s, "precip6hr" ); - get_filtered_nc_data(in_precip12hr_var, precip12hr_arr, buf_size, i_hdr_s, "precip12hr" ); - get_filtered_nc_data(in_precip10min_var, precip10min_arr, buf_size, i_hdr_s, "precip10min" ); - get_filtered_nc_data(in_precip1min_var, precip1min_arr, buf_size, i_hdr_s, "precip1min" ); - get_filtered_nc_data(in_windDir10_var, windDir10_arr, buf_size, i_hdr_s, "windDir10" ); - get_filtered_nc_data(in_windSpeed10_var, windSpeed10_arr, buf_size, i_hdr_s, "windSpeed10" ); + get_nc_qty_data(&in_temperatureQty_var, temperatureQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_dewpointQty_var, dewpointQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_relHumidityQty_var, relHumidityQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_stationPressureQty_var, stationPressureQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_seaLevelPressureQty_var, seaLevelPressureQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_windDirQty_var, windDirQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_windSpeedQty_var, windSpeedQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_windGustQty_var, windGustQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_visibilityQty_var, visibilityQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_precipRateQty_var, precipRateQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_solarRadiationQty_var, solarRadiationQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_seaSurfaceTempQty_var, seaSurfaceTempQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_totalColumnPWVQty_var, totalColumnPWVQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_soilTemperatureQty_var, soilTemperatureQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_minTemp24HourQty_var, minTemp24HourQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_maxTemp24HourQty_var, maxTemp24HourQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_precip3hrQty_var, precip3hrQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_precip6hrQty_var, precip6hrQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_precip12hrQty_var, precip12hrQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_precip10minQty_var, precip10minQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_precip1minQty_var, precip1minQty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_windDir10Qty_var, windDir10Qty_arr, buf_size, i_hdr_s); + get_nc_qty_data(&in_windSpeed10Qty_var, windSpeed10Qty_arr, buf_size, i_hdr_s); + + get_filtered_nc_data_opt(in_temperature_var, temperature_arr, buf_size, i_hdr_s, "temperature" , optional_vars); + get_filtered_nc_data_opt(in_dewpoint_var, dewpoint_arr, buf_size, i_hdr_s, "dewpoint" , optional_vars); + get_filtered_nc_data_opt(in_relHumidity_var, relHumidity_arr, buf_size, i_hdr_s, "relHumidity" , optional_vars); + get_filtered_nc_data_opt(in_stationPressure_var, stationPressure_arr, buf_size, i_hdr_s, "stationPressure" , optional_vars); + get_filtered_nc_data_opt(in_seaLevelPressure_var, seaLevelPressure_arr, buf_size, i_hdr_s, "seaLevelPressure", optional_vars); + get_filtered_nc_data_opt(in_windDir_var, windDir_arr, buf_size, i_hdr_s, "windDir" , optional_vars); + get_filtered_nc_data_opt(in_windSpeed_var, windSpeed_arr, buf_size, i_hdr_s, "windSpeed" , optional_vars); + get_filtered_nc_data_opt(in_windGust_var, windGust_arr, buf_size, i_hdr_s, "windGust" , optional_vars); + get_filtered_nc_data_opt(in_visibility_var, visibility_arr, buf_size, i_hdr_s, "visibility" , optional_vars); + get_filtered_nc_data_opt(in_precipRate_var, precipRate_arr, buf_size, i_hdr_s, "precipRate" , optional_vars); + get_filtered_nc_data_opt(in_solarRadiation_var, solarRadiation_arr, buf_size, i_hdr_s, "solarRadiation" , optional_vars); + get_filtered_nc_data_opt(in_seaSurfaceTemp_var, seaSurfaceTemp_arr, buf_size, i_hdr_s, "seaSurfaceTemp" , optional_vars); + get_filtered_nc_data_opt(in_totalColumnPWV_var, totalColumnPWV_arr, buf_size, i_hdr_s, "totalColumnPWV" , optional_vars); + get_filtered_nc_data_opt(in_soilTemperature_var, soilTemperature_arr, buf_size, i_hdr_s, "soilTemperature" , optional_vars); + get_filtered_nc_data_opt(in_minTemp24Hour_var, minTemp24Hour_arr, buf_size, i_hdr_s, "minTemp24Hour" , optional_vars); + get_filtered_nc_data_opt(in_maxTemp24Hour_var, maxTemp24Hour_arr, buf_size, i_hdr_s, "maxTemp24Hour" , optional_vars); + get_filtered_nc_data_opt(in_precip3hr_var, precip3hr_arr, buf_size, i_hdr_s, "precip3hr" , optional_vars); + get_filtered_nc_data_opt(in_precip6hr_var, precip6hr_arr, buf_size, i_hdr_s, "precip6hr" , optional_vars); + get_filtered_nc_data_opt(in_precip12hr_var, precip12hr_arr, buf_size, i_hdr_s, "precip12hr" , optional_vars); + get_filtered_nc_data_opt(in_precip10min_var, precip10min_arr, buf_size, i_hdr_s, "precip10min" , optional_vars); + get_filtered_nc_data_opt(in_precip1min_var, precip1min_arr, buf_size, i_hdr_s, "precip1min" , optional_vars); + get_filtered_nc_data_opt(in_windDir10_var, windDir10_arr, buf_size, i_hdr_s, "windDir10" , optional_vars); + get_filtered_nc_data_opt(in_windSpeed10_var, windSpeed10_arr, buf_size, i_hdr_s, "windSpeed10" , optional_vars); + dim[1] = hdr_sid_len; get_nc_data(&in_hdr_sid_var, (char *)hdr_sid_arr, dim, cur); diff --git a/met/src/tools/other/madis2nc/madis2nc_conf_info.cc b/met/src/tools/other/madis2nc/madis2nc_conf_info.cc index cf6c2af280..0f0145eed8 100644 --- a/met/src/tools/other/madis2nc/madis2nc_conf_info.cc +++ b/met/src/tools/other/madis2nc/madis2nc_conf_info.cc @@ -52,43 +52,47 @@ void Madis2NcConfInfo::init_from_scratch() void Madis2NcConfInfo::clear() { + mesonet_optional_vars.clear(); _version.clear(); } //////////////////////////////////////////////////////////////////////// void Madis2NcConfInfo::read_config(const string &default_filename, - const string &user_filename) + const string &user_filename) { - // Read the config file constants + // Read the config file constants - _conf.read(replace_path(config_const_filename).c_str()); - - // Read the default config file + _conf.read(replace_path(config_const_filename).c_str()); - _conf.read(replace_path(default_filename).c_str()); + // Read the default config file - // Read the user config file + _conf.read(replace_path(default_filename).c_str()); - _conf.read(user_filename.c_str()); + // Read the user config file - // Process the configuration file + _conf.read(user_filename.c_str()); - process_config(); + // Process the configuration file - return; + process_config(); + + return; } //////////////////////////////////////////////////////////////////////// void Madis2NcConfInfo::process_config() { - - _version = parse_conf_version(&_conf); - check_met_version(_version.c_str()); - _timeSummaryInfo = parse_conf_time_summary(&_conf); - + _version = parse_conf_version(&_conf); + check_met_version(_version.c_str()); + + StringArray sa = _conf.lookup_string_array(conf_key_mesonet_opt_var, false); + for(int i=0; i + + &MET_BIN;/madis2nc + \ + &DATA_DIR_OBS;/madis/mesonet/mesonet_20170101_0000.nc \ + &OUTPUT_DIR;/madis2nc/mesonet_20170101_0000_F000.nc \ + -type mesonet -mask_grid G207 -v 2 + + + &OUTPUT_DIR;/madis2nc/mesonet_20170101_0000_F000.nc + + + &MET_BIN;/madis2nc \