Skip to content

Commit

Permalink
Bugfix 391 fixed values (#392)
Browse files Browse the repository at this point in the history 
* Issue #391 utils for added fixed_vars_vals_input support

* Issue #391 tests for plot correctness. Includes data, config file, expected plot, and expected data points

* fix formatting

* Issue #391 add support to read in fixed_vars_vals_input setting

* Issue #391 perform filtering if fixed_vars_vals_input has any values defined

* issue #391 update to support the legacy format expected by METviewer when R was in use or a more simplified format for defining the fixed_vars_vals_input setting

* Issue #391 clean up tests, update config files

* issue #391 update the syntax for setting the fixed_vars_vals_input

* add issue to branch name to activate documentation workflow

* issue 391 updates to query, log when query returns an empty string

* issue #391 don't filter when the query string is an empty string

* Issue #391 remove call to non-existent logger in the create_query function

---------

Co-authored-by: Julie Prestopnik <[email protected]>
  • Loading branch information
@bikegeek @jprestop
bikegeek and jprestop authored Oct 16, 2023
1 parent cfd2335 commit 1c14aec
Show file tree
Hide file tree
Showing 14 changed files with 1,311 additions and 155 deletions.
1 change: 1 addition & 0 deletions .github/workflows/documentation.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:
- feature_*
- main_*
- bugfix_*
- issue_*
paths:
- docs/**
pull_request:
Expand Down
237 changes: 140 additions & 97 deletions metplotpy/plots/line/line.py
View file

Large diffs are not rendered by default.

129 changes: 113 additions & 16 deletions metplotpy/plots/line/line_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ def __init__(self, parameters: dict) -> None:
# used by METviewer
self.points_path = self.get_config_value('points_path')

# Associated fixed values corresponding the METviewer Fixed Values
# to identify groups over which statistics are calculated.

self.fixed_vars_vals = self._get_fixed_vars_vals()

# plot parameters
self.grid_on = self._get_bool('grid_on')
self.plot_width = self.calculate_plot_dimension('plot_width', 'pixels')
Expand Down Expand Up @@ -200,6 +205,83 @@ def _get_fcst_vars(self, index):

return fcst_var_val_dict

def _get_fixed_vars_vals(self) -> dict:
"""
Retrieve a list of the inner keys (name of the variables to hold 'fixed') to
the fixed_vars_vals dictionary. These values correspond to the Fixed
Values set in the METviewer user interface. In the YAML configuration file,
the value(s) are set under the fixed_vars_vals_input setting:
fixed_vars_vals_input:
- fcst_lev:
- 'Z0'
- vx_mask:
- 'CONUS'
- 'WEST'
- fcst_thresh:
- '>0.0'
- '>=5.1'
Also supports the 'legacy' format (used by METviewer when R script was
employed):
fixed_vars_vals_input:
- fcst_lev:
- fcst_lev_0:
- 'Z0'
- vx_mask:
- vx_mask_1:
- 'CONUS'
- 'EAST'
Generate a new dictionary where the value of the inner key is
associated to the outer key.
Mixing of the two formats is also supported.
Args:
Returns:
updated_fixed_vars_vals_dict:
a dictionary of the keys and values associated with the
fixed_vars_vals_input setting in the configuration file (corresponds to
the METviewer Fixed Variable setting(s) ).
"""

fixed_vars_vals_dict = self.get_config_value('fixed_vars_vals_input')
if len(fixed_vars_vals_dict) == 0:
# If user hasn't specified anything in the fixed_var_vals_input setting,
# return an empty dictionary.
return {}
else:
# Use the outer dictionary key and the inner dictionary value
# Retrieve all the inner keys, then their corresponding values and
# associate those inner values with the outer key in a new dictionary.
outer_keys = fixed_vars_vals_dict.keys()
updated_fixed_vars_vals_dict = {}
inner_exists = False
for key in outer_keys:
try:
inner_keys = fixed_vars_vals_dict[key].keys()
except AttributeError:
# No inner dictionary, format looks like:
#
updated_fixed_vars_vals_dict[key] = fixed_vars_vals_dict[key]
else:
# inner dictionary, assign the value corresponding to the
# inner dictionary to the key of the outer dictionary.
for inner_key in inner_keys:
updated_fixed_vars_vals_dict[key] = fixed_vars_vals_dict[
key][inner_key]


return updated_fixed_vars_vals_dict


def _get_mode(self) -> list:
"""
Retrieve all the modes. Convert mode names from
Expand Down Expand Up @@ -294,15 +376,17 @@ def _get_plot_stat(self) -> str:
to represent this combination. Acceptable values are sum, mean, and median.
Returns:
stat_to_plot: one of the following values for the plot_stat: MEAN, MEDIAN, or SUM
stat_to_plot: one of the following values for the plot_stat: MEAN,
MEDIAN, or SUM
"""

accepted_stats = ['MEAN', 'MEDIAN', 'SUM']
stat_to_plot = self.get_config_value('plot_stat').upper()

if stat_to_plot not in accepted_stats:
raise ValueError("An unsupported statistic was set for the plot_stat setting. "
" Supported values are sum, mean, and median.")
raise ValueError(
"An unsupported statistic was set for the plot_stat setting. "
" Supported values are sum, mean, and median.")
return stat_to_plot

def _config_consistency_check(self) -> bool:
Expand Down Expand Up @@ -347,8 +431,10 @@ def _get_plot_ci(self) -> list:
Args:
Returns:
list of values to indicate whether or not to plot the confidence interval for
a particular series, and which confidence interval (bootstrap or normal).
list of values to indicate whether or not to plot the confidence
interval for
a particular series, and which confidence interval (bootstrap or
normal).
"""
plot_ci_list = self.get_config_value('plot_ci')
Expand All @@ -360,22 +446,27 @@ def _get_plot_ci(self) -> list:
if ci_setting not in constants.ACCEPTABLE_CI_VALS:
raise ValueError("A plot_ci value is set to an invalid value. "
"Accepted values are (case insensitive): "
"None, met_prm, or boot. Please check your config file.")
"None, met_prm, or boot. Please check your config "
"file.")

return self.create_list_by_series_ordering(ci_settings_list)

def _get_user_legends(self, legend_label_type: str = '') -> list:
"""
Retrieve the text that is to be displayed in the legend at the bottom of the plot.
Retrieve the text that is to be displayed in the legend at the bottom of the
plot.
Each entry corresponds to a series.
Args:
@parm legend_label_type: The legend label, such as 'Performance' that indicates
the type of series line. Used when the user hasn't
@parm legend_label_type: The legend label, such as 'Performance'
that indicates
the type of series line. Used when the user
hasn't
indicated a legend.
Returns:
a list consisting of the series label to be displayed in the plot legend.
a list consisting of the series label to be displayed in the plot
legend.
"""

Expand All @@ -397,7 +488,8 @@ def _get_user_legends(self, legend_label_type: str = '') -> list:
# index of the legend
legend_idx = idx + num_series_y1

if legend_idx >= len(all_user_legends) or all_user_legends[legend_idx].strip() == '':
if legend_idx >= len(all_user_legends) or all_user_legends[
legend_idx].strip() == '':
# user did not provide the legend - create it
legend_list.append(' '.join(map(str, ser_components)))
else:
Expand All @@ -410,7 +502,8 @@ def _get_user_legends(self, legend_label_type: str = '') -> list:
for idx, ser_components in enumerate(self.get_config_value('derived_series_1')):
# index of the legend
legend_idx = idx + num_series_y1 + num_series_y2
if legend_idx >= len(all_user_legends) or all_user_legends[legend_idx].strip() == '':
if legend_idx >= len(all_user_legends) or all_user_legends[
legend_idx].strip() == '':
# user did not provide the legend - create it
legend_list.append(utils.get_derived_curve_name(ser_components))
else:
Expand All @@ -422,7 +515,8 @@ def _get_user_legends(self, legend_label_type: str = '') -> list:
# index of the legend
legend_idx = idx + num_series_y1 + num_series_y2 \
+ len(self.get_config_value('derived_series_1'))
if legend_idx >= len(all_user_legends) or all_user_legends[legend_idx].strip() == '':
if legend_idx >= len(all_user_legends) or all_user_legends[
legend_idx].strip() == '':
# user did not provide the legend - create it
legend_list.append(utils.get_derived_curve_name(ser_components))
else:
Expand All @@ -433,12 +527,14 @@ def _get_user_legends(self, legend_label_type: str = '') -> list:

def get_series_y(self, axis: int) -> list:
"""
Creates an array of series components (excluding derived) tuples for the specified y-axis
Creates an array of series components (excluding derived) tuples for the
specified y-axis
:param axis: y-axis (1 or 2)
:return: an array of series components tuples
"""
if self.get_config_value('series_val_' + str(axis)) is not None:
all_fields_values_orig = all_fields_values_orig = self.get_config_value('series_val_' + str(axis)).copy()
all_fields_values_orig = all_fields_values_orig = self.get_config_value(
'series_val_' + str(axis)).copy()
else:
all_fields_values_orig = {}
all_fields_values = {}
Expand All @@ -464,7 +560,8 @@ def _get_all_series_y(self, axis: int) -> list:

# add derived series if exist
if self.get_config_value('derived_series_' + str(axis)):
all_series = all_series + self.get_config_value('derived_series_' + str(axis))
all_series = all_series + self.get_config_value(
'derived_series_' + str(axis))

return all_series

Expand Down
24 changes: 21 additions & 3 deletions metplotpy/plots/line/line_series.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ def __init__(self, config, idx: int, input_data, series_list: list,
self.series_list = series_list
self.series_name = series_name
super().__init__(config, idx, input_data, y_axis)
# Retrieve any fixed variables


self.logger = metplotpy.plots.util.get_common_logger(config.log_level,
config.log_filename)

Expand Down Expand Up @@ -130,6 +133,19 @@ def _create_series_points(self) -> dict:
series_data_1 = None
series_data_2 = None

# first subset based on the fixed variable values if any exist
if len(self.config.fixed_vars_vals) > 0:

query_string = metplotpy.plots.util.create_query(self.input_data,
self.config.fixed_vars_vals)
if query_string == " ":
filtered_df = self.input_data
else:
filtered_df = self.input_data.query(query_string)
else:
# Otherwise use the original input data
filtered_df = self.input_data

# different ways to subset data for normal and derived series
if self.series_name[-1] not in utils.OPERATION_TO_SIGN.keys():
# this is a normal series
Expand All @@ -156,26 +172,28 @@ def _create_series_points(self) -> dict:
filter_list[i] = int(filter_val)
elif utils.is_string_strictly_float(filter_val):
filter_list[i] = float(filter_val)

all_filters.append((self.input_data[field].isin(filter_list)))
all_filters.append((filtered_df[field].isin(filter_list)))

# filter by provided indy

# Duck typing is different in Python 3.6 and Python 3.8, for
# Python 3.8 and above, explicitly type cast the self.input_data[self.config.indy_var]
# Panda Series object to 'str' if the list of indy_vals are of str type.
# This will ensure we are doing str to str comparisons.

if isinstance(self.config.indy_vals[0], str):
indy_var_series = self.input_data[self.config.indy_var].astype(str)
indy_var_series = filtered_df[self.config.indy_var].astype(str)
else:
# The Panda series is as it was originally coded.
indy_var_series = self.input_data[self.config.indy_var]
indy_var_series = filtered_df[self.config.indy_var]

all_filters.append(indy_var_series.isin(self.config.indy_vals))

# use numpy to select the rows where any record evaluates to True
mask = np.array(all_filters).all(axis=0)
self.series_data = self.input_data.loc[mask]
self.series_data = filtered_df.loc[mask]

# sort data by date/time - needed for CI calculations
if 'fcst_lead' in self.series_data.columns:
Expand Down
Loading

0 comments on commit 1c14aec

Please sign in to comment.