From 52b64d10e74736da65d907bd433d079418d28932 Mon Sep 17 00:00:00 2001 From: Alexander <122119488+energinet-ajw@users.noreply.github.com> Date: Fri, 24 Jan 2025 13:28:34 +0100 Subject: [PATCH] fix: Incorrect settlement group in electrical heating scenario tests (#156) --- .../consumption_metering_point_periods_v1.py | 2 +- .../consumption_metering_point_periods_v1.csv | 8 +++---- .../consumption_metering_point_periods_v1.csv | 4 ++-- .../consumption_metering_point_periods_v1.csv | 2 +- .../consumption_metering_point_periods_v1.csv | 2 +- .../consumption_metering_point_periods_v1.csv | 6 ++--- .../consumption_metering_point_periods_v1.csv | 4 ++-- .../consumption_metering_point_periods_v1.csv | 2 +- .../consumption_metering_point_periods_v1.csv | 2 +- .../tests/utils/measurements_utils.py | 24 +++++++------------ 10 files changed, 24 insertions(+), 32 deletions(-) diff --git a/source/electrical_heating/src/electrical_heating/infrastructure/electricity_market/schemas/consumption_metering_point_periods_v1.py b/source/electrical_heating/src/electrical_heating/infrastructure/electricity_market/schemas/consumption_metering_point_periods_v1.py index 19ff7502..62967834 100644 --- a/source/electrical_heating/src/electrical_heating/infrastructure/electricity_market/schemas/consumption_metering_point_periods_v1.py +++ b/source/electrical_heating/src/electrical_heating/infrastructure/electricity_market/schemas/consumption_metering_point_periods_v1.py @@ -15,7 +15,7 @@ t.StructField("has_electrical_heating", t.BooleanType(), not nullable), # # 2 | 3 | 4 | 5 | 6 | 99 | NULL - t.StructField("net_settlement_group", t.IntegerType(), not nullable), + t.StructField("net_settlement_group", t.IntegerType(), nullable), # # Settlement month is 1st of January for all consumption with electrical heating except for # net settlement group 6, where the date is the scheduled meter reading date. diff --git a/source/electrical_heating/tests/scenario_tests/wip_different_periods_hit_different_limits/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_different_periods_hit_different_limits/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 720e07ca..f0838a62 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_different_periods_hit_different_limits/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_different_periods_hit_different_limits/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,5 +1,5 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-11-30T23:00:00Z;;Two periods because crossing years. Period 1: December 2023 (31 days). Period 2: Jan. 2024 until today (full limit) -170000000000000202;true;2;1;2023-11-21T23:00:00Z;2024-02-29T23:00:00Z;Two periods because crossing years. Period 1: November 21st to end of year 2023 (9+31 days). Period 2: Jan-feb 2024 (31+29 days) -170000000000000203;true;2;1;2024-02-29T23:00:00Z;2024-03-01T23:00:00Z;One period of one day. -170000000000000204;true;2;1;2024-02-29T23:00:00Z;;One period of 10 months - Mar-Dec 2024 (306 days) +170000000000000201;true;;1;2023-11-30T23:00:00Z;;Two periods because crossing years. Period 1: December 2023 (31 days). Period 2: Jan. 2024 until today (full limit) +170000000000000202;true;;1;2023-11-21T23:00:00Z;2024-02-29T23:00:00Z;Two periods because crossing years. Period 1: November 21st to end of year 2023 (9+31 days). Period 2: Jan-feb 2024 (31+29 days) +170000000000000203;true;;1;2024-02-29T23:00:00Z;2024-03-01T23:00:00Z;One period of one day. +170000000000000204;true;;1;2024-02-29T23:00:00Z;;One period of 10 months - Mar-Dec 2024 (306 days) diff --git a/source/electrical_heating/tests/scenario_tests/wip_filtering_and_period_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_filtering_and_period_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 46dabbd0..35c09eab 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_filtering_and_period_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_filtering_and_period_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,3 +1,3 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;2024-01-03T23:00:00Z;Previously period_to_date was null here. Now there is a new period: Limit = 3/366*4000 = 32.786 -170000000000000201;true;2;1;2024-01-03T23:00:00Z;;Limit for the rest of the year: 363/366*4000 = 3967.213 +170000000000000201;true;;1;2023-12-31T23:00:00Z;2024-01-03T23:00:00Z;Previously period_to_date was null here. Now there is a new period: Limit = 3/366*4000 = 32.786 +170000000000000201;true;;1;2024-01-03T23:00:00Z;;Limit for the rest of the year: 363/366*4000 = 3967.213 diff --git a/source/electrical_heating/tests/scenario_tests/wip_filtering_measurement_data_updated/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_filtering_measurement_data_updated/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 1e480687..a561345e 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_filtering_measurement_data_updated/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_filtering_measurement_data_updated/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,2 +1,2 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;; +170000000000000201;true;;1;2023-12-31T23:00:00Z;; diff --git a/source/electrical_heating/tests/scenario_tests/wip_filtering_simplest_case/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_filtering_simplest_case/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index bba7ed56..cac57b1c 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_filtering_simplest_case/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_filtering_simplest_case/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,2 +1,2 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;;Generates output because there is consumption data and an electrical heating child metering point \ No newline at end of file +170000000000000201;true;;1;2023-12-31T23:00:00Z;;Generates output because there is consumption data and an electrical heating child metering point \ No newline at end of file diff --git a/source/electrical_heating/tests/scenario_tests/wip_gaps/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_gaps/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 42140ab7..ec262b5c 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_gaps/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_gaps/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,4 +1,4 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;2024-02-29T23:00:00Z;Case 1: Period 1 (Jan-Feb). Limit: 60/366*4000 = 655.738 -170000000000000201;true;2;1;2024-04-30T22:00:00Z;2024-07-31T22:00:00Z;Case 1: Period 2 (May-Jul). Limit: 92/366*4000 = 1005.464 -170000000000000201;true;2;1;2024-10-31T23:00:00Z;;Case 1: Period 3 (Nov-Dec). Limit: 61/366*4000 = 666.667 \ No newline at end of file +170000000000000201;true;;1;2023-12-31T23:00:00Z;2024-02-29T23:00:00Z;Case 1: Period 1 (Jan-Feb). Limit: 60/366*4000 = 655.738 +170000000000000201;true;;1;2024-04-30T22:00:00Z;2024-07-31T22:00:00Z;Case 1: Period 2 (May-Jul). Limit: 92/366*4000 = 1005.464 +170000000000000201;true;;1;2024-10-31T23:00:00Z;;Case 1: Period 3 (Nov-Dec). Limit: 61/366*4000 = 666.667 \ No newline at end of file diff --git a/source/electrical_heating/tests/scenario_tests/wip_non_matching_child_periods/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_non_matching_child_periods/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 1ddb7067..038c0be1 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_non_matching_child_periods/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_non_matching_child_periods/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,3 +1,3 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;;EH consumption MP with no end date for has_electrical_heating -170000000000000202;true;2;1;2023-12-31T23:00:00Z;;EH consumption MP without child MP +170000000000000201;true;;1;2023-12-31T23:00:00Z;;EH consumption MP with no end date for has_electrical_heating +170000000000000202;true;;1;2023-12-31T23:00:00Z;;EH consumption MP without child MP diff --git a/source/electrical_heating/tests/scenario_tests/wip_summertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_summertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 7c16f3a6..cebc1ff4 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_summertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_summertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,2 +1,2 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;;Generates output because there is consumption data and an electrical heating child metering point +170000000000000201;true;;1;2023-12-31T23:00:00Z;;Generates output because there is consumption data and an electrical heating child metering point diff --git a/source/electrical_heating/tests/scenario_tests/wip_wintertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/scenario_tests/wip_wintertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv index 7c16f3a6..cebc1ff4 100644 --- a/source/electrical_heating/tests/scenario_tests/wip_wintertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv +++ b/source/electrical_heating/tests/scenario_tests/wip_wintertime_change/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -1,2 +1,2 @@ metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date;#comment -170000000000000201;true;2;1;2023-12-31T23:00:00Z;;Generates output because there is consumption data and an electrical heating child metering point +170000000000000201;true;;1;2023-12-31T23:00:00Z;;Generates output because there is consumption data and an electrical heating child metering point diff --git a/source/electrical_heating/tests/utils/measurements_utils.py b/source/electrical_heating/tests/utils/measurements_utils.py index 7165ba64..2591d70c 100644 --- a/source/electrical_heating/tests/utils/measurements_utils.py +++ b/source/electrical_heating/tests/utils/measurements_utils.py @@ -1,17 +1,15 @@ -from testcommon.dataframes import assert_schema +import pyspark.sql.functions as F +from pyspark.sql import DataFrame, SparkSession from pyspark.sql.types import ArrayType +from testcommon.dataframes import assert_schema + from electrical_heating.infrastructure.measurements_bronze.schemas.measurements_bronze_v1 import ( measurements_bronze_v1, point, ) -from pyspark.sql import DataFrame, SparkSession -import pyspark.sql.functions as F -def create_measurements_dataframe( - spark: SparkSession, - measurements: DataFrame -) -> DataFrame: +def create_measurements_dataframe(spark: SparkSession, measurements: DataFrame) -> DataFrame: measurements = measurements.withColumn( "points", F.from_json(F.col("points"), ArrayType(point)), @@ -22,20 +20,14 @@ def create_measurements_dataframe( "start_datetime", F.to_timestamp(F.col("start_datetime"), "yyyy-MM-dd HH:mm:ss"), ) - .withColumn( - "end_datetime", F.to_timestamp(F.col("end_datetime"), "yyyy-MM-dd HH:mm:ss") - ) + .withColumn("end_datetime", F.to_timestamp(F.col("end_datetime"), "yyyy-MM-dd HH:mm:ss")) .withColumn( "transaction_creation_datetime", - F.to_timestamp( - F.col("transaction_creation_datetime"), "yyyy-MM-dd HH:mm:ss" - ), + F.to_timestamp(F.col("transaction_creation_datetime"), "yyyy-MM-dd HH:mm:ss"), ) ) - measurements = spark.createDataFrame( - measurements.rdd, schema=measurements_bronze_v1, verifySchema=True - ) + measurements = spark.createDataFrame(measurements.rdd, schema=measurements_bronze_v1, verifySchema=True) assert_schema(measurements.schema, measurements_bronze_v1)