From 502b12306a9d9eb6891f4589cf655774ddd81d70 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Mon, 11 Mar 2024 15:23:41 +0100 Subject: [PATCH 01/11] GSYE-689: Add LiveProfileTypes enum --- gsy_framework/read_user_profile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gsy_framework/read_user_profile.py b/gsy_framework/read_user_profile.py index a12d9970..74112527 100644 --- a/gsy_framework/read_user_profile.py +++ b/gsy_framework/read_user_profile.py @@ -47,6 +47,12 @@ class InputProfileTypes(Enum): ENERGY_KWH = 4 +class LiveProfileTypes(Enum): + """Enum for type of live data profiles""" + MEASUREMENT = 0 + FORECAST = 1 + + def _str_to_datetime(time_string, time_format) -> DateTime: """ Converts time_string into a pendulum (DateTime) object that either takes the global start date From e0be7b0594a9dc1cb9786fc707e55e3bd4856b5b Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Tue, 19 Mar 2024 12:27:22 +0100 Subject: [PATCH 02/11] GSYE-689: Add no-live-data enum to LiveProfileTypes --- gsy_framework/read_user_profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gsy_framework/read_user_profile.py b/gsy_framework/read_user_profile.py index 74112527..4a186fa7 100644 --- a/gsy_framework/read_user_profile.py +++ b/gsy_framework/read_user_profile.py @@ -49,8 +49,9 @@ class InputProfileTypes(Enum): class LiveProfileTypes(Enum): """Enum for type of live data profiles""" - MEASUREMENT = 0 + NO_LIVE_DATA = 0 FORECAST = 1 + MEASUREMENT = 2 def _str_to_datetime(time_string, time_format) -> DateTime: From 956e292ebfd835deb190ea3f96bbb29e0ef3016d Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Thu, 18 Apr 2024 14:34:15 +0200 Subject: [PATCH 03/11] GSYE-689: If an empty profile has been provided, return early in read_arbitrary_profile --- gsy_framework/read_user_profile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gsy_framework/read_user_profile.py b/gsy_framework/read_user_profile.py index 4a186fa7..7faba11b 100644 --- a/gsy_framework/read_user_profile.py +++ b/gsy_framework/read_user_profile.py @@ -332,6 +332,8 @@ def read_arbitrary_profile(profile_type: InputProfileTypes, :param current_timestamp: :return: a mapping from time to profile values """ + if not input_profile: + return {} profile = _read_from_different_sources_todict(input_profile, current_timestamp=current_timestamp) profile_time_list = list(profile.keys()) From 2cf303a1d4b0707d6e452bb3d75a6ccbbb77f789 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Thu, 18 Apr 2024 14:34:50 +0200 Subject: [PATCH 04/11] GSY-689: If an empty profile has been provided, skip validating in ProfileValidator --- gsy_framework/validators/profile_validator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gsy_framework/validators/profile_validator.py b/gsy_framework/validators/profile_validator.py index 4131037c..54c17d76 100644 --- a/gsy_framework/validators/profile_validator.py +++ b/gsy_framework/validators/profile_validator.py @@ -11,7 +11,7 @@ class ProfileValidator: def __init__( self, profile: Dict[DateTime, float], start_time: Optional[DateTime] = None, end_time: Optional[DateTime] = None, slot_length: Optional[timedelta] = None): - assert len(profile) > 0, "profile is empty" + self.profile = OrderedDict(profile) self.start_time = start_time if start_time else self._profile_start_time self.end_time = end_time if end_time else self._profile_end_time @@ -19,6 +19,8 @@ def __init__( def validate(self): """Validate if profile corresponds to the start_time, end_time and slot_length setting.""" + if not self.profile: + return if self.slot_length: self._validate_slot_length() else: From fac7b21db758960f0e3d0cc4928feeb15096d1a9 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Thu, 18 Apr 2024 14:55:35 +0200 Subject: [PATCH 05/11] GSYE-689: Fix empty profile handling in read_arbitrary_profile --- gsy_framework/read_user_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsy_framework/read_user_profile.py b/gsy_framework/read_user_profile.py index 7faba11b..a104bdcc 100644 --- a/gsy_framework/read_user_profile.py +++ b/gsy_framework/read_user_profile.py @@ -332,7 +332,7 @@ def read_arbitrary_profile(profile_type: InputProfileTypes, :param current_timestamp: :return: a mapping from time to profile values """ - if not input_profile: + if input_profile in [{}, None]: return {} profile = _read_from_different_sources_todict(input_profile, current_timestamp=current_timestamp) From dfb4d2eda56d6da33d87da5c6eac7e30f8986108 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Thu, 18 Apr 2024 15:24:03 +0200 Subject: [PATCH 06/11] GSYE-689: Add test in order to increase coverage --- gsy_framework/validators/profile_validator.py | 2 ++ tests/test_read_user_profile.py | 7 +++++++ tests/test_validators/test_profile_validator.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/gsy_framework/validators/profile_validator.py b/gsy_framework/validators/profile_validator.py index 54c17d76..ff48b37e 100644 --- a/gsy_framework/validators/profile_validator.py +++ b/gsy_framework/validators/profile_validator.py @@ -13,6 +13,8 @@ def __init__( end_time: Optional[DateTime] = None, slot_length: Optional[timedelta] = None): self.profile = OrderedDict(profile) + if not self.profile: + return self.start_time = start_time if start_time else self._profile_start_time self.end_time = end_time if end_time else self._profile_end_time self.slot_length: timedelta = slot_length diff --git a/tests/test_read_user_profile.py b/tests/test_read_user_profile.py index 6de05f6f..b113cd73 100644 --- a/tests/test_read_user_profile.py +++ b/tests/test_read_user_profile.py @@ -192,3 +192,10 @@ def test_resample_energy_profile_performs_correctly_for_equal_resolutions(): duration(hours=4), datetime(2021, 1, 25, 0, 0)) assert result_profile == input_profile + + @staticmethod + def test_read_arbitrary_profile_returns_early_for_empty_profiles(): + assert read_arbitrary_profile(InputProfileTypes.POWER_W, {}) == {} + assert read_arbitrary_profile(InputProfileTypes.POWER_W, None) == {} + assert len(read_arbitrary_profile(InputProfileTypes.POWER_W, 0)) == 96 + assert set(read_arbitrary_profile(InputProfileTypes.POWER_W, 0).values()) == {0} diff --git a/tests/test_validators/test_profile_validator.py b/tests/test_validators/test_profile_validator.py index 58cf0fe6..48cf6f82 100644 --- a/tests/test_validators/test_profile_validator.py +++ b/tests/test_validators/test_profile_validator.py @@ -1,4 +1,5 @@ from copy import copy +from unittest.mock import Mock import pytest @@ -8,6 +9,7 @@ class TestProfileValidator: + # pylint: disable=protected-access @staticmethod @pytest.mark.parametrize("input_profile, end_time", [ @@ -87,3 +89,15 @@ def test_profile_validator_fails_if_gap_in_profile_no_slot_length(input_profile) del profile[gap_time] with pytest.raises(AssertionError): ProfileValidator(profile=profile).validate() + + @staticmethod + def test_profile_validator_returns_early_if_empty_profile(): + validator = ProfileValidator(profile={}) + validator._validate_slot_length = Mock() + validator._get_and_validate_time_diffs = Mock() + validator._validate_start_end_date = Mock() + validator.slot_length = Mock() + assert validator.validate() is None + validator._validate_slot_length.assert_not_called() + validator._get_and_validate_time_diffs.assert_not_called() + validator._validate_start_end_date.assert_not_called() From d58b3f71335ba2e84c73824f3aa37304ea795509 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Thu, 18 Apr 2024 15:32:17 +0200 Subject: [PATCH 07/11] GSYE-689: Fix test --- tests/test_read_user_profile.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_read_user_profile.py b/tests/test_read_user_profile.py index b113cd73..5d3c1335 100644 --- a/tests/test_read_user_profile.py +++ b/tests/test_read_user_profile.py @@ -195,7 +195,13 @@ def test_resample_energy_profile_performs_correctly_for_equal_resolutions(): @staticmethod def test_read_arbitrary_profile_returns_early_for_empty_profiles(): + original_slot_length = GlobalConfig.slot_length + original_sim_duration = GlobalConfig.sim_duration + GlobalConfig.slot_length = duration(hours=1) + GlobalConfig.sim_duration = duration(hours=4) assert read_arbitrary_profile(InputProfileTypes.POWER_W, {}) == {} assert read_arbitrary_profile(InputProfileTypes.POWER_W, None) == {} - assert len(read_arbitrary_profile(InputProfileTypes.POWER_W, 0)) == 96 + assert len(read_arbitrary_profile(InputProfileTypes.POWER_W, 0)) == 4 assert set(read_arbitrary_profile(InputProfileTypes.POWER_W, 0).values()) == {0} + GlobalConfig.slot_length = original_slot_length + GlobalConfig.sim_duration = original_sim_duration From bbd7224fbfa7149620b2509e3197f35062637bc5 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Fri, 19 Apr 2024 12:42:04 +0200 Subject: [PATCH 08/11] GSYE-689: For CNs only buffer the current market slot in read_arbitrary_profile --- gsy_framework/utils.py | 2 +- tests/test_read_user_profile.py | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/gsy_framework/utils.py b/gsy_framework/utils.py index 50633810..2de572bc 100644 --- a/gsy_framework/utils.py +++ b/gsy_framework/utils.py @@ -98,7 +98,7 @@ def generate_market_slot_list(start_timestamp=None): No input arguments, required input is only handled by a preconfigured GlobalConfig @return: List with market slot datetimes """ - time_span = (duration(days=PROFILE_EXPANSION_DAYS) + time_span = (GlobalConfig.slot_length if GlobalConfig.is_canary_network() else min(GlobalConfig.sim_duration, duration(days=PROFILE_EXPANSION_DAYS))) time_span += duration(hours=ConstSettings.FutureMarketSettings.FUTURE_MARKET_DURATION_HOURS) diff --git a/tests/test_read_user_profile.py b/tests/test_read_user_profile.py index 5d3c1335..0d79d95b 100644 --- a/tests/test_read_user_profile.py +++ b/tests/test_read_user_profile.py @@ -18,7 +18,7 @@ import pytest from pendulum import datetime, duration, today -from gsy_framework.constants_limits import PROFILE_EXPANSION_DAYS, TIME_ZONE, GlobalConfig +from gsy_framework.constants_limits import TIME_ZONE, GlobalConfig from gsy_framework.enums import ConfigurationType from gsy_framework.read_user_profile import ( InputProfileTypes, _fill_gaps_in_profile, _generate_slot_based_zero_values_dict_from_profile, @@ -122,16 +122,8 @@ def test_read_profile_for_player(): def test_read_arbitrary_profile_returns_correct_profile_in_canary_network( set_is_canary_network): set_is_canary_network(True) - market_maker_rate = 30 - GlobalConfig.sim_duration = duration(hours=3) - expected_last_time_slot = today(tz=TIME_ZONE).add(days=PROFILE_EXPANSION_DAYS - 1, - hours=23, minutes=45) - mmr = read_arbitrary_profile(InputProfileTypes.IDENTITY, market_maker_rate) - assert list(mmr.keys())[-1] == expected_last_time_slot - GlobalConfig.sim_duration = duration(hours=30) - expected_last_time_slot = today(tz=TIME_ZONE).add(days=PROFILE_EXPANSION_DAYS - 1, - hours=23, minutes=45) - mmr = read_arbitrary_profile(InputProfileTypes.IDENTITY, market_maker_rate) + expected_last_time_slot = today(tz=TIME_ZONE) + mmr = read_arbitrary_profile(InputProfileTypes.IDENTITY, 30) assert list(mmr.keys())[-1] == expected_last_time_slot @staticmethod From edf2aed1b1ec02c4b5e998eb1a28c57d010a05aa Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Fri, 19 Apr 2024 12:42:51 +0200 Subject: [PATCH 09/11] GSYE-689: Do not calculate bills in SCM when there is no data received from the simulation e.g. for the root area --- gsy_framework/sim_results/scm/bills.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gsy_framework/sim_results/scm/bills.py b/gsy_framework/sim_results/scm/bills.py index a3ab74bb..c2ba44c9 100644 --- a/gsy_framework/sim_results/scm/bills.py +++ b/gsy_framework/sim_results/scm/bills.py @@ -55,6 +55,7 @@ def _empty_bills_dict() -> Dict: } def update(self, area_result_dict, core_stats, current_market_slot): + # print(area_result_dict, core_stats, current_market_slot) if not self._has_update_parameters( area_result_dict, core_stats, current_market_slot): return @@ -63,7 +64,8 @@ def update(self, area_result_dict, core_stats, current_market_slot): if not area.get("children"): continue - if "bills" not in core_stats[area["uuid"]]: + if "bills" not in core_stats[area["uuid"]] or not core_stats[area["uuid"]]["bills"]: + print("skipping", core_stats[area["uuid"]]) continue if area["uuid"] not in self.bills_redis_results: From 01c2f7fac420a596a69934685860e5404e6f507d Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Fri, 19 Apr 2024 12:43:35 +0200 Subject: [PATCH 10/11] GSYE-689: Add measurement uuid parameters to acro schema --- .../schema/avro_schemas/launch_simulation_scenario.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gsy_framework/schema/avro_schemas/launch_simulation_scenario.json b/gsy_framework/schema/avro_schemas/launch_simulation_scenario.json index 11f611b0..e1656cd0 100644 --- a/gsy_framework/schema/avro_schemas/launch_simulation_scenario.json +++ b/gsy_framework/schema/avro_schemas/launch_simulation_scenario.json @@ -157,6 +157,7 @@ ]}, {"name": "power_profile", "type": ["null", "string"]}, {"name": "power_profile_uuid", "type": ["null", "string"]}, + {"name": "power_measurement_uuid", "type": ["null", "string"]}, {"name": "capacity_kW", "type": ["null", "float"]}, {"name": "tilt", "type": ["null", "float"]}, {"name": "azimuth", "type": ["null", "float"]}, @@ -195,6 +196,7 @@ {"name": "energy_rate_decrease_per_update", "type": ["null", "float"]}, {"name": "power_profile", "type": ["null", "string"]}, {"name": "power_profile_uuid", "type": ["null", "string"]}, + {"name": "power_measurement_uuid", "type": ["null", "string"]}, {"name": "update_interval", "type": ["null", "int"]}, {"name": "fit_to_limit", "type": ["null","boolean"]}, {"name": "capacity_kW", "type": ["null", "float"]}, @@ -227,6 +229,7 @@ {"name": "target_device_kpi", "type": ["null", "int"]}, {"name": "daily_load_profile", "type": ["null", "string"]}, {"name": "daily_load_profile_uuid", "type": ["null", "string"]}, + {"name": "daily_load_measurement_uuid", "type": ["null", "string"]}, {"name": "use_market_maker_rate", "type": ["null", "boolean"]}, {"name": "forecast_stream_enabled", "type": ["null", "boolean"]}, {"name": "forecast_stream_id", "type": ["null", "string"]}, @@ -315,6 +318,7 @@ ]}, {"name": "prosumption_kWh_profile", "type": ["null", "string"]}, {"name": "prosumption_kWh_profile_uuid", "type": ["null", "string"]}, + {"name": "prosumption_kWh_measurement_uuid", "type": ["null", "string"]}, {"name": "use_market_maker_rate", "type": ["null", "boolean"]}, {"name": "allow_external_connection", "type": ["null", "boolean"]}, {"name": "display_type", "type": "string"}, @@ -338,6 +342,7 @@ ]}, {"name": "consumption_kWh_profile", "type": ["null", "string"]}, {"name": "consumption_kWh_profile_uuid", "type": ["null", "string"]}, + {"name": "consumption_kWh_measurement_uuid", "type": ["null", "string"]}, {"name": "use_market_maker_rate", "type": ["null", "boolean"]}, {"name": "allow_external_connection", "type": ["null", "boolean"]}, {"name": "display_type", "type": "string"}, @@ -363,6 +368,7 @@ ]}, {"name": "smart_meter_profile", "type": ["null", "string"]}, {"name": "smart_meter_profile_uuid", "type": ["null", "string"]}, + {"name": "smart_meter_measurement_uuid", "type": ["null", "string"]}, {"name": "final_buying_rate", "type": ["null", "float"]}, {"name": "initial_buying_rate", "type": ["null", "float"]}, {"name": "final_selling_rate", "type": ["null", "float"]}, @@ -398,9 +404,11 @@ {"name": "initial_temp_C", "type": "float"}, {"name": "external_temp_C_profile", "type": ["null", "string"]}, {"name": "external_temp_C_profile_uuid", "type": ["null", "string"]}, + {"name": "external_temp_C_measurement_uuid", "type": ["null", "string"]}, {"name": "tank_volume_l", "type": "float"}, {"name": "consumption_kWh_profile", "type": ["null", "string"]}, {"name": "consumption_kWh_profile_uuid", "type": ["null", "string"]}, + {"name": "consumption_kWh_measurement_uuid", "type": ["null", "string"]}, {"name": "source_type", "type": "int"}, {"name": "final_buying_rate", "type": ["null", "float"]}, {"name": "initial_buying_rate", "type": ["null", "float"]}, From 4a17ec5bdfb92204580c232e24b188bed2962676 Mon Sep 17 00:00:00 2001 From: hannesdiedrich Date: Mon, 22 Apr 2024 12:04:20 +0200 Subject: [PATCH 11/11] GSYE-689: Remove unneeded prints --- gsy_framework/sim_results/scm/bills.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gsy_framework/sim_results/scm/bills.py b/gsy_framework/sim_results/scm/bills.py index c2ba44c9..c4f52743 100644 --- a/gsy_framework/sim_results/scm/bills.py +++ b/gsy_framework/sim_results/scm/bills.py @@ -55,7 +55,6 @@ def _empty_bills_dict() -> Dict: } def update(self, area_result_dict, core_stats, current_market_slot): - # print(area_result_dict, core_stats, current_market_slot) if not self._has_update_parameters( area_result_dict, core_stats, current_market_slot): return @@ -65,7 +64,6 @@ def update(self, area_result_dict, core_stats, current_market_slot): continue if "bills" not in core_stats[area["uuid"]] or not core_stats[area["uuid"]]["bills"]: - print("skipping", core_stats[area["uuid"]]) continue if area["uuid"] not in self.bills_redis_results: