From 497403c159dff772837942646ea51155c0d4d5aa Mon Sep 17 00:00:00 2001 From: Darryl Melander Date: Fri, 21 May 2021 19:12:28 -0600 Subject: [PATCH] When parsing RTS-GMLC files, filter which scalar reserve requirements are included in a model based on the simulation type and what's listed in the Reserve_Products row of simulation_objects.csv. Note that this only applies to scalar reserve requirements (the ones listed in reserves.csv). It does not filter timeseries by reserve type; if a timeseries is listed in timeseries_pointers.csv for a given reserve, the timeseries is included in the model even if the reserve type isn't listed in simulation_objects.csv. If simulation_objects.csv does not have a Reserve_Products row then all reserve products are accepted for both DAY_AHEAD and REAL_TIME models. --- egret/parsers/rts_gmlc/_reserves.py | 37 +++++++++++ egret/parsers/rts_gmlc/parsed_cache.py | 22 ++++++- egret/parsers/rts_gmlc/parser.py | 88 +++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 12 deletions(-) diff --git a/egret/parsers/rts_gmlc/_reserves.py b/egret/parsers/rts_gmlc/_reserves.py index d25dd12c..f4b8bb11 100644 --- a/egret/parsers/rts_gmlc/_reserves.py +++ b/egret/parsers/rts_gmlc/_reserves.py @@ -9,6 +9,8 @@ from __future__ import annotations +from typing import NamedTuple, Optional, Sequence + reserve_name_map = { 'Spin_Up': 'spinning_reserve_requirement', 'Reg_Up': 'regulation_up_requirement', @@ -25,3 +27,38 @@ def is_valid_reserve_name(name:str, model_dict:dict=None): res, area = name.split('_R', 1) return (res in reserve_name_map) and \ ((model_dict is None) or (area in model_dict['elements']['area'])) + +class ScalarReserveValue(NamedTuple): + ''' A reserve type, scope, and scalar value. + + If area_name is None, this is a global reserve value. + ''' + reserve_type: str + area_name: Optional[str] + value: float + +class ScalarReserveData(): + ''' Scalar reserve values that should only be applied to one type of model. + ''' + def __init__(self, + da_scalars: Sequence[ScalarReserveValue], + rt_scalars: Sequence[ScalarReserveValue]): + self._da_scalars = da_scalars + self._rt_scalars = rt_scalars + + @property + def da_scalars(self) -> Sequence[ScalarReserveValue]: + ''' Scalar reserve values that only apply to DAY_AHEAD models + ''' + return self._da_scalars + + @property + def rt_scalars(self) -> Sequence[ScalarReserveValue]: + ''' Scalar reserve values that only apply to REAL_TIME models + ''' + return self._rt_scalars + + def get_simulation_reserves(self, simulation_type:str) -> Sequence[ScalarReserveValue]: + ''' Get scalar reserve values that only apply the requested simulation type + ''' + return self._rt_scalars if simulation_type == 'REAL_TIME' else self._da_scalars diff --git a/egret/parsers/rts_gmlc/parsed_cache.py b/egret/parsers/rts_gmlc/parsed_cache.py index c59ef8ab..ecd8eb39 100644 --- a/egret/parsers/rts_gmlc/parsed_cache.py +++ b/egret/parsers/rts_gmlc/parsed_cache.py @@ -20,7 +20,7 @@ from egret.data.model_data import ModelData -from ._reserves import reserve_name_map +from ._reserves import reserve_name_map, ScalarReserveData class ParsedCache(): @@ -28,7 +28,8 @@ def __init__(self, model_skeleton:dict, begin_time:datetime, end_time:datetime, minutes_per_day_ahead_period:int, minutes_per_real_time_period:int, timeseries_data:DataFrame, - load_participation_factors:Dict[str,float]): + load_participation_factors:Dict[str,float], + scalar_reserve_data:ScalarReserveData): self.skeleton = model_skeleton self.begin_time = begin_time self.end_time = end_time @@ -47,6 +48,8 @@ def __init__(self, model_skeleton:dict, cur_sim = self.timeseries_df['Simulation'].iat[i] self._first_indices[cur_sim] = i + self.scalar_reserve_data = scalar_reserve_data + def generate_model(self, simulation_type:str, begin_time:datetime, end_time:datetime) -> ModelData: """ Create a new model populated with requested data @@ -87,6 +90,7 @@ def populate_skeleton_with_data(self, skeleton_dict:dict, simulation_type:str, #Because pandas includes the end of a range, reduce our end time by one second end_time = end_time - timedelta(seconds=1) + self._insert_scalar_reserve_data(skeleton_dict, simulation_type) self._process_timeseries_data(skeleton_dict, simulation_type, begin_time, end_time) self._insert_system_data(skeleton_dict, simulation_type, begin_time, end_time) @@ -183,3 +187,17 @@ def _insert_system_data(self, md:dict, simulation_type:str, sample_df = df.iat[self._first_indices[simulation_type], df.columns.get_loc('Series')] dates = sample_df[begin_time:end_time].index md['system']['time_keys'] = [dt.strftime('%Y-%m-%d %H:%M') for dt in dates] + + def _insert_scalar_reserve_data(self, md:dict, simulation_type:str): + ''' Insert scalar reserve values into the model dict + ''' + system = md['system'] + areas = md['elements']['area'] + + reserve_list = self.scalar_reserve_data.get_simulation_reserves(simulation_type) + for res in reserve_list: + if res.area_name is None: + target_dict = system + else: + target_dict = areas[res.area_name] + target_dict[reserve_name_map[res.reserve_type]] = res.value diff --git a/egret/parsers/rts_gmlc/parser.py b/egret/parsers/rts_gmlc/parser.py index c1938c6b..6bb9ceed 100644 --- a/egret/parsers/rts_gmlc/parser.py +++ b/egret/parsers/rts_gmlc/parser.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Dict, Union, Optional, Tuple + from typing import Dict, Union, Optional, Tuple, Set import sys import os.path @@ -24,7 +24,7 @@ import egret.data.model_data as md from .parsed_cache import ParsedCache -from ._reserves import is_valid_reserve_name, reserve_name_map +from ._reserves import is_valid_reserve_name, reserve_name_map, ScalarReserveData, ScalarReserveValue def create_ModelData(rts_gmlc_dir:str, begin_time:Union[datetime,str], end_time:Union[datetime,str], @@ -133,6 +133,8 @@ def parse_to_cache(rts_gmlc_dir:str, data_start, data_end = _get_data_date_range(metadata_df) + constant_reserve_data = _get_scalar_reserve_data(rts_gmlc_dir, metadata_df, model_data) + begin_time, end_time = _parse_datetimes_if_strings(begin_time, end_time) # TODO: Validate begin_time and end_time. # Do we want to enforce that they fall within the data date range? @@ -146,7 +148,7 @@ def parse_to_cache(rts_gmlc_dir:str, return ParsedCache(model_data, begin_time, end_time, minutes_per_period['DAY_AHEAD'], minutes_per_period['REAL_TIME'], - timeseries_df, load_participation_factors) + timeseries_df, load_participation_factors, constant_reserve_data) def _read_metadata(base_dir:str) -> pd.DataFrame: @@ -324,7 +326,7 @@ def _read_2D_timeseries_file(file_name:str, minutes_per_period:int, # Create and return a new 1-column DataFrame return pd.DataFrame({column_name: s}) -def _create_rtsgmlc_skeleton(rts_gmlc_dir:str): +def _create_rtsgmlc_skeleton(rts_gmlc_dir:str) -> dict: """ Creates a data dictionary from the RTS-GMLC data files, without loading hourly data @@ -632,26 +634,92 @@ def valid_output_pcts(): elements["generator"][name] = gen_dict gen_df = None - # Add the reserves + return model_data + +def _get_scalar_reserve_data(base_dir:str, metadata_df:df, model_dict:dict) -> ScalarReserveData: + # Store scalar reserve values as stored in the input + # + # Scalar reserve values that apply to both simulation types are stored in the + # passed in model dict. Scalar values that vary depending on model type are stored + # in the returned ScalarReserveData. + + da_scalar_reserves, rt_scalar_reserves = _identify_allowed_scalar_reserve_types(metadata_df) + shared_reserves = da_scalar_reserves.intersection(rt_scalar_reserves) + + # Collect constant scalar reserves + da_scalars = [] + rt_scalars = [] reserve_df = pd.read_csv(os.path.join(base_dir,'reserves.csv')) + system = model_dict['system'] + areas = model_dict['elements']['area'] for idx,row in reserve_df.iterrows(): res_name = row['Reserve Product'] req = float(row['Requirement (MW)']) if res_name in reserve_name_map: target_dict = system + area_name = None else: # reserve name must be _R. # split into type and area res_name, area_name = res_name.split("_R", 1) - if area_name not in elements['area']: - # Skip areas not referenced elsewhere + if res_name not in reserve_name_map: + logger.warning(f"Skipping reserve for unrecognized reserve type '{res_name}'") + continue + if area_name not in areas: + logger.warning(f"Skipping reserve for unrecognized area '{area_name}'") continue - target_dict = elements['area'][area_name] + target_dict = areas[area_name] - target_dict[reserve_name_map[res_name]] = req + if res_name in shared_reserves: + # If it applies to both types, save it in the skeleton + target_dict[reserve_name_map[res_name]] = req + elif res_name in da_scalar_reserves: + # If it applies to just day-ahead, save to DA cache + da_scalars.append(ScalarReserveValue(res_name, area_name, req)) + elif res_name in rt_scalar_reserves: + # If it applies to just real-time, save to RT cache + rt_scalars.append(ScalarReserveValue(res_name, area_name, req)) - return model_data + return ScalarReserveData(da_scalars, rt_scalars) + +def _identify_allowed_scalar_reserve_types(metadata_df:df) -> Tuple[Set[str], Set[str]]: + ''' Return a list of reserve types that apply to each type of model (DA and RT). + + Arguments + --------- + metadata_df:df + The contents of simulation_objects.csv in a DataFrame + + Returns + ------- + Returns a tuple with two lists of strings, one list for day-ahead models, + and another list for real-time models. Each list holds the names of reserve + categories whose scalar reserve values (if specified in reserves.csv) should + be applied to that type of model. + + (day_ahead_reserves, rt_reserves) + day_ahead_reserves: Sequence[str] + The names of reserve categories whose scalar values apply to day-ahead models + rt_reserves: Sequence[str] + The names of reserve categories whose scalar values apply to real-time models + ''' + if not 'Reserve_Products' in metadata_df.index: + # By default, accept all reserve types in both types of model + all_reserves = set(reserve_name_map.keys()) + return (all_reserves, all_reserves) + + row = metadata_df.loc['Reserve_Products'] + def parse_reserves(which:str) -> Sequence[str]: + all = row[which] + if type(all) is not str: + return {} + # strip off parentheses, if present + all = all.strip('()') + return set(s.strip() for s in all.split(',')) + + return (parse_reserves('DAY_AHEAD'), parse_reserves('REAL_TIME')) + def _compute_bus_load_participation_factors(model_data): '''