getting rts-gmlc working

prescient/data/data_provider.py

@@ -45,8 +45,22 @@ def negotiate_data_frequency(self, desired_frequency_minutes:int):
         pass
 
     @abstractmethod
-    def get_initial_model(self, options:Options, num_time_steps:int) -> EgretModel:
-        ''' Get a model ready to be populated with data
+    def get_initial_forecast_model(self, options:Options, num_time_steps:int) -> EgretModel:
+        ''' Get a forecast model ready to be populated with data
+
+        Returns
+        -------
+        A model object populated with static system information, such as
+        buses and generators, and with time series arrays that are large
+        enough to hold num_time_steps entries.
+
+        Initial values in time time series do not have meaning.
+        '''
+        pass
+
+    @abstractmethod
+    def get_initial_actuals_model(self, options:Options, num_time_steps:int) -> EgretModel:
+        ''' Get an actuals model ready to be populated with data
 
         Returns
         -------

prescient/data/providers/dat_data_provider.py

@@ -59,7 +59,13 @@ def negotiate_data_frequency(self, desired_frequency_minutes:int):
         # This provider can only return one value every 60 minutes.
         return 60
 
-    def get_initial_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+    def get_initial_forecast_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+        return self._get_initial_model(options, num_time_steps, minutes_per_timestep)
+
+    def get_initial_actuals_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+        return self._get_initial_model(options, num_time_steps, minutes_per_timestep)
+
+    def _get_initial_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
         ''' Get a model ready to be populated with data
 
         Returns
@@ -236,7 +242,7 @@ def _populate_with_forecastable_data(self, options:Options,
 
             dat = identify_dat(day)
 
-            for src, target in forecast_helper.get_forecastables(dat, model):
+            for src, target in _get_forecastables(dat, model):
                 target[step_index] = src[src_step_index]
 
 
@@ -307,3 +313,33 @@ def _recurse_copy_with_time_series_length(root:Dict[str, Any], time_count:int) -
         else:
             new_node[key] = copy.deepcopy(att)
     return new_node
+
+def _get_forecastables(*models: EgretModel) -> Iterable[ Tuple[MutableSequence[float]] ]:
+    ''' Get all data that are predicted by forecasting, for any number of models.
+        Specialization of this for the dat data provider with fixed checks
+
+    The iterable returned by this function yields tuples containing one list from each model
+    passed to the function.  Each tuple of lists corresponds to one type of data that is included
+    in forecast predictions, such as loads on a particular bus, or limits on a renewable generator.
+    The lengths of the lists matches the number of time steps present in the underlying models.
+    Modifying list values modifies the underlying model.
+    '''
+    # Renewables limits
+    model1 = models[0]
+    for gen, gdata1 in model1.elements('generator', generator_type=('renewable','virtual')):
+        if isinstance(gdata1['p_min'], dict):
+            yield tuple(m.data['elements']['generator'][gen]['p_min']['values'] for m in models)
+        if isinstance(gdata1['p_max'], dict):
+            yield tuple(m.data['elements']['generator'][gen]['p_max']['values'] for m in models)
+        if 'p_cost' in gdata1 and isinstance(gdata1['p_cost'], dict):
+            yield tuple(m.data['elements']['generator'][gen]['p_cost']['values'] for m in models)
+
+    # Loads
+    for bus, bdata1 in model1.elements('load'):
+        yield tuple(m.data['elements']['load'][bus]['p_load']['values'] for m in models)
+        if 'p_price' in bdata1 and isinstance(bdata1['p_price'], dict):
+            yield tuple(m.data['elements']['load'][bus]['p_price']['values'] for m in models)
+
+    # Reserve requirement
+    if 'reserve_requirement' in model1.data['system']:
+        yield tuple(m.data['system']['reserve_requirement']['values'] for m in models)

prescient/data/providers/gmlc_data_provider.py

@@ -18,6 +18,7 @@
 import copy
 
 from egret.parsers import rts_gmlc_parser as parser
+from egret.parsers.rts_gmlc._reserves import reserve_name_map
 from egret.data.model_data import ModelData as EgretModel
 
 from ..data_provider import DataProvider
@@ -60,7 +61,13 @@ def negotiate_data_frequency(self, desired_frequency_minutes:int):
         else:
             return native_frequency
 
-    def get_initial_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+    def get_initial_actuals_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+        return self._get_initial_model("REAL_TIME", options, num_time_steps, minutes_per_timestep)
+
+    def get_initial_forecast_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+        return self._get_initial_model("DAY_AHEAD", options, num_time_steps, minutes_per_timestep)
+
+    def _get_initial_model(self, sim_type:str, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
         ''' Get a model ready to be populated with data
 
         Returns
@@ -75,7 +82,7 @@ def get_initial_model(self, options:Options, num_time_steps:int, minutes_per_tim
         data['system']['time_period_length_minutes'] = minutes_per_timestep
         data['system']['time_keys'] = [str(i) for i in range(1,num_time_steps+1)]
         md = EgretModel(data)
-        forecast_helper.ensure_forecastable_storage(num_time_steps, md)
+        self._ensure_forecastable_storage(sim_type, num_time_steps, md)
         return md
 
     def populate_initial_state_data(self, options:Options,
@@ -176,7 +183,6 @@ def populate_with_actuals(self, options:Options,
         '''
         self._populate_with_forecastable_data('REAL_TIME', start_time, num_time_periods, time_period_length_minutes, model)
 
-
     def _populate_with_forecastable_data(self,
                                          sim_type:str,
                                          start_time:datetime,
@@ -201,6 +207,65 @@ def _populate_with_forecastable_data(self,
             dt = start_time + i*delta
             time_labels[i] = dt.strftime('%Y-%m-%d %H:%M')
 
+    def _get_forecastable_locations(self, simulation_type:str, md:EgretModel):
+        df = self._cache.timeseries_df
+
+        system = md.data['system']
+        loads = md.data['elements']['load']
+        generators = md.data['elements']['generator']
+        areas = md.data['elements']['area']
+
+        # Go through each timeseries value for this simulation type
+        for i in range(self._cache._first_indices[simulation_type], len(df)):
+            if df.iat[i, df.columns.get_loc('Simulation')] != simulation_type:
+                break
+
+            category = df.iat[i, df.columns.get_loc('Category')]
+
+            if category == 'Generator':
+                gen_name = df.iat[i, df.columns.get_loc('Object')]
+                param = df.iat[i, df.columns.get_loc('Parameter')]
+
+                if param == 'PMin MW':
+                    yield (generators[gen_name], 'p_min')
+                elif param == 'PMax MW':
+                    yield (generators[gen_name], 'p_max')
+                else:
+                    raise ValueError(f"Unexpected generator timeseries data: {param}")
+
+            elif category == 'Area':
+                area_name = df.iat[i, df.columns.get_loc('Object')]
+                param = df.iat[i, df.columns.get_loc('Parameter')]
+                assert(param == "MW Load")
+                for l_d in loads.values():
+                    # Skip loads from other areas
+                    if l_d['area'] != area_name:
+                        continue
+                    yield (l_d, 'p_load')
+                    yield (l_d, 'q_load')
+
+            elif category == 'Reserve':
+                res_name = df.iat[i, df.columns.get_loc('Object')]
+                if res_name in reserve_name_map:
+                    yield (system, reserve_name_map[res_name])
+                else:
+                    # reserve name must be <type>_R<area>,
+                    # split into type and area
+                    res_name, area_name = res_name.split("_R", 1)
+                    yield (areas[area_name], reserve_name_map[res_name])
+
+    def _ensure_forecastable_storage(self, sim_type:str, num_entries:int, model:EgretModel) -> None:
+        """ Ensure that the model has an array allocated for every type of forecastable data
+        """
+        for data, key in self._get_forecastable_locations(sim_type, model):
+            if (key not in data or \
+                type(data[key]) is not dict or \
+                data[key]['data_type'] != 'time_series' or \
+                len(data[key]['values'] != num_entries)
+               ):
+                data[key] = { 'data_type': 'time_series',
+                              'values': [None]*num_entries}
+
 def _recurse_copy_at_ratio(src:dict[str, Any], target:dict[str, Any], ratio:int) -> None:
     ''' Copy every Nth value from a src dict's time_series values into corresponding arrays in a target dict.
     '''

prescient/data/providers/shortcut_data_provider.py

@@ -83,7 +83,13 @@ def negotiate_data_frequency(self, desired_frequency_minutes:int):
         else:
             return native_frequency
 
-    def get_initial_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+    def get_initial_forecast_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+        return self._get_initial_model(options, num_time_steps, minutes_per_timestep)
+
+    def get_initial_actuals_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
+        return self._get_initial_model(options, num_time_steps, minutes_per_timestep)
+
+    def _get_initial_model(self, options:Options, num_time_steps:int, minutes_per_timestep:int) -> EgretModel:
         ''' Get a model ready to be populated with data
         Returns
         -------

prescient/data/simulation_state/mutable_simulation_state.py

@@ -26,8 +26,8 @@ class MutableSimulationState(SimulationState):
     '''
 
     def __init__(self):
-        self._forecasts = []
-        self._actuals = []
+        self._forecasts = {}
+        self._actuals = {}
         self._commits = {}
 
         self._init_gen_state = {}
@@ -49,7 +49,9 @@ def __init__(self):
     @property
     def timestep_count(self) -> int:
         ''' The number of timesteps we have data for '''
-        return len(self._forecasts[0]) if len(self._forecasts) > 0 else 0
+        for _ in self._forecasts.values():
+            return len(_)
+        return 0
 
     @property
     def minutes_per_step(self) -> int:
@@ -73,18 +75,17 @@ def get_initial_state_of_charge(self, s:S) -> float:
         ''' Get state of charge in the previous time period '''
         return self._init_soc[s]
 
-    def get_current_actuals(self) -> Iterable[float]:
-        ''' Get the current actual value for each forecastable.
+    def get_current_actuals(self, forecastable:str) -> float:
+        ''' Get the current actual value for forecastable
 
         This is the actual value for the current time period (time index 0).
         Values are returned in the same order as forecast_helper.get_forecastables,
         but instead of returning arrays it returns a single value.
         '''
-        for forecastable in self._actuals:
-            yield forecastable[0]
+        return self._actuals[forecastable][0]
 
-    def get_forecasts(self) -> Iterable[Sequence[float]]:
-        ''' Get the forecast values for each forecastable 
+    def get_forecasts(self, forecastable:str) -> Sequence[float]:
+        ''' Get the forecast values for forecastable
 
         This is very similar to forecast_helper.get_forecastables(); the 
         function yields an array per forecastable, in the same order as
@@ -93,18 +94,16 @@ def get_forecasts(self) -> Iterable[Sequence[float]]:
         Note that the value at index 0 is the forecast for the current time,
         not the actual value for the current time.
         '''
-        for forecastable in self._forecasts:
-            yield forecastable
+        return self._forecasts[forecastable]
 
-    def get_future_actuals(self) -> Iterable[Sequence[float]]:
-        ''' Warning: Returns actual values for the current time AND FUTURE TIMES.
+    def get_future_actuals(self, forecastable:str) -> Sequence[float]:
+        ''' Warning: Returns actual values of forecastable for the current time AND FUTURE TIMES.
 
         Be aware that this function returns information that is not yet known!
         The function lets you peek into the future.  Future actuals may be used
         by some (probably unrealistic) algorithm options, such as 
         '''
-        for forecastable in self._actuals:
-            yield forecastable
+        return self._actuals[forecastable]
 
     def apply_ruc(self, options, ruc:RucModel) -> None:
         ''' Incorporate a RUC instance into the current state.
@@ -192,14 +191,14 @@ def apply_sced(self, options, sced) -> None:
         self._simulation_minute += self._sced_frequency
 
         while self._next_forecast_pop_minute <= self._simulation_minute:
-            for value_deque in self._forecasts:
+            for value_deque in self._forecasts.values():
                 value_deque.popleft()
             for value_deque in self._commits.values():
                 value_deque.popleft()
             self._next_forecast_pop_minute += self._minutes_per_forecast_step
 
         while self._simulation_minute >= self._next_actuals_pop_minute:
-            for value_deque in self._actuals:
+            for value_deque in self._actuals.values():
                 value_deque.popleft()
             self._next_actuals_pop_minute += self._minutes_per_actuals_step
 
@@ -225,17 +224,17 @@ def _save_forecastables(options, ruc, where_to_store, steps_per_hour):
     max_length = steps_per_hour*(ruc_delay + options.ruc_horizon)
 
     # Save all forecastables, in forecastable order
-    for idx, (new_ruc_vals,) in enumerate(get_forecastables(ruc)):
+    for key, new_ruc_vals in get_forecastables(ruc):
         if first_ruc:
             # append to storage array
             forecast = deque(maxlen=max_length)
-            where_to_store.append(forecast)
+            where_to_store[key] = forecast
         else:
-            forecast = where_to_store[idx]
+            forecast = where_to_store[key]
 
             # Pop until the first "ruc_delay" items are the only items in the list
             for _ in range(len(forecast) - steps_per_hour*ruc_delay):
                 forecast.pop()
 
         # Put the new values into the value queue
-        forecast.extend(new_ruc_vals) 
+        forecast.extend(new_ruc_vals)

prescient/data/simulation_state/simulation_state.py

@@ -47,8 +47,8 @@ def get_initial_state_of_charge(self, s:S):
         pass
 
     @abstractmethod
-    def get_current_actuals(self) -> Iterable[float]:
-        ''' Get the current actual value for each forecastable.
+    def get_current_actuals(self, forecastable:str) -> float:
+        ''' Get the current actual value for forecastable
 
         This is the actual value for the current time period (time index 0).
         Values are returned in the same order as forecast_helper.get_forecastables,
@@ -57,8 +57,8 @@ def get_current_actuals(self) -> Iterable[float]:
         pass
 
     @abstractmethod
-    def get_forecasts(self) -> Iterable[Sequence[float]]:
-        ''' Get the forecast values for each forecastable 
+    def get_forecasts(self, forecastable:str) -> Sequence[float]:
+        ''' Get the forecast values for forecastable
 
         This is very similar to forecast_helper.get_forecastables(); the 
         function yields an array per forecastable, in the same order as
@@ -70,8 +70,8 @@ def get_forecasts(self) -> Iterable[Sequence[float]]:
         pass
 
     @abstractmethod
-    def get_future_actuals(self) -> Iterable[Sequence[float]]:
-        ''' Warning: Returns actual values of forecastables for the current time AND FUTURE TIMES.
+    def get_future_actuals(self, forecastable:str) -> Sequence[float]:
+        ''' Warning: Returns actual values of forecastable for the current time AND FUTURE TIMES.
 
         Be aware that this function returns information that is not yet known!
         The function lets you peek into the future.  Future actuals may be used

prescient/data/simulation_state/state_with_offset.py

@@ -82,18 +82,17 @@ def get_initial_state_of_charge(self, s:S):
         ''' Get state of charge in the previous time period '''
         return self._init_soc[s]
 
-    def get_current_actuals(self) -> Iterable[float]:
-        ''' Get the current actual value for each forecastable.
+    def get_current_actuals(self, forecastable:str) -> float:
+        ''' Get the current actual value for forecastable
 
         This is the actual value for the current time period (time index 0).
         Values are returned in the same order as forecast_helper.get_forecastables,
         but instead of returning arrays it returns a single value.
         '''
-        for actual in self._parent.get_future_actuals():
-            yield actual[self._offset]
+        return self._parent.get_future_actuals(forecastable)[self._offset]
 
-    def get_forecasts(self) -> Iterable[Sequence[float]]:
-        ''' Get the forecast values for each forecastable 
+    def get_forecasts(self, forecastable:str) -> Sequence[float]:
+        ''' Get the forecast values for forecastable
 
         This is very similar to forecast_helper.get_forecastables(); the 
         function yields an array per forecastable, in the same order as
@@ -102,19 +101,15 @@ def get_forecasts(self) -> Iterable[Sequence[float]]:
         Note that the value at index 0 is the forecast for the current time,
         not the actual value for the current time.
         '''
-        for forecast in self._parent.get_forecasts():
-            # Copy the relevent portion to a new array
-            portion = list(itertools.islice(forecast, self._offset, None))
-            yield portion
+        # Copy the relevent portion to a new array
+        return list(itertools.islice(self._parent.get_forecasts(forecastable), self._offset, None))
 
-    def get_future_actuals(self) -> Iterable[Sequence[float]]:
-        ''' Warning: Returns actual values for the current time AND FUTURE TIMES.
+    def get_future_actuals(self, forecastable:str) -> Sequence[float]:
+        ''' Warning: Returns actual values of forecastable for the current time AND FUTURE TIMES.
 
         Be aware that this function returns information that is not yet known!
         The function lets you peek into the future.  Future actuals may be used
         by some (probably unrealistic) algorithm options, such as 
         '''
-        for future in self._parent.get_future_actuals():
-            # Copy the relevent portion to a new array
-            portion = list(itertools.islice(future, self._offset, None))
-            yield portion
+        # Copy the relevent portion to a new array
+        return list(itertools.islice(self._parent.get_future_actuals(forecastable), self._offset, None))