From dc4aec609761411ee3a203c169351057b3869bc9 Mon Sep 17 00:00:00 2001 From: IW-SS <133041871+IW-SS@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:09:13 +0530 Subject: [PATCH] Source connector for ERCOT System Level Data (#583) * Adding ERCOT Daily Load Source Signed-off-by: Shivam Saxena * Fixing spaces in components.md Signed-off-by: Shivam Saxena * Fixing spaces in mkdocs.md Signed-off-by: Shivam Saxena * Fixing Pandas Compatibility Signed-off-by: Shivam Saxena * Putting HTML Raw response in the test file for ERCOT Signed-off-by: Shivam Saxena * Fixing Sonar Code Smells Signed-off-by: Shivam Saxena * Code cleanups Signed-off-by: Shivam Saxena --------- Signed-off-by: Shivam Saxena --- .../sources/spark/iso/ercot_daily_load_iso.md | 1 + docs/sdk/pipelines/components.md | 1 + mkdocs.yml | 5 +- .../pipelines/_pipeline_utils/iso.py | 17 + .../_pipeline_utils/temp_cert_files.py | 27 + .../pipelines/sources/spark/iso/__init__.py | 1 + .../pipelines/sources/spark/iso/base_iso.py | 11 +- .../sources/spark/iso/ercot_daily_load_iso.py | 218 +++ .../data/ercot_daily_load_actual_expected.csv | 25 + .../data/ercot_daily_load_actual_sample1.zip | Bin 0 -> 1330 bytes .../data/ercot_daily_load_actual_sample2.zip | Bin 0 -> 500 bytes .../ercot_daily_load_forecast_expected.csv | 193 +++ .../ercot_daily_load_forecast_sample1.zip | Bin 0 -> 8316 bytes .../spark/iso/test_ercot_daily_load_iso.py | 1233 +++++++++++++++++ 14 files changed, 1724 insertions(+), 8 deletions(-) create mode 100644 docs/sdk/code-reference/pipelines/sources/spark/iso/ercot_daily_load_iso.md create mode 100644 src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/temp_cert_files.py create mode 100644 src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/ercot_daily_load_iso.py create mode 100644 tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_expected.csv create mode 100644 tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_sample1.zip create mode 100644 tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_sample2.zip create mode 100644 tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_expected.csv create mode 100644 tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_sample1.zip create mode 100644 tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/test_ercot_daily_load_iso.py diff --git a/docs/sdk/code-reference/pipelines/sources/spark/iso/ercot_daily_load_iso.md b/docs/sdk/code-reference/pipelines/sources/spark/iso/ercot_daily_load_iso.md new file mode 100644 index 000000000..aea5520fc --- /dev/null +++ b/docs/sdk/code-reference/pipelines/sources/spark/iso/ercot_daily_load_iso.md @@ -0,0 +1 @@ +::: src.sdk.python.rtdip_sdk.pipelines.sources.spark.iso.ercot_daily_load_iso \ No newline at end of file diff --git a/docs/sdk/pipelines/components.md b/docs/sdk/pipelines/components.md index b353d9761..40f7bd98f 100644 --- a/docs/sdk/pipelines/components.md +++ b/docs/sdk/pipelines/components.md @@ -45,6 +45,7 @@ Sources are components that connect to source systems and extract data from them |[PJM Historical Load ISO](../code-reference/pipelines/sources/spark/iso/pjm_historical_load_iso.md) ||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:| |[CAISO Daily Load ISO](../code-reference/pipelines/sources/spark/iso/caiso_daily_load_iso.md) ||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:| |[CAISO Historical Load ISO](../code-reference/pipelines/sources/spark/iso/caiso_historical_load_iso.md) ||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:| +|[ERCOT Daily Load ISO](../code-reference/pipelines/sources/spark/iso/ercot_daily_load_iso.md) ||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:| |[Weather Forecast API V1](../code-reference/pipelines/sources/spark/the_weather_company/weather_forecast_api_v1.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:| |[Weather Forecast API V1 Multi](../code-reference/pipelines/sources/spark/the_weather_company/weather_forecast_api_v1_multi.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:| !!! note "Note" diff --git a/mkdocs.yml b/mkdocs.yml index 88a269d4b..164934d8c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -171,6 +171,7 @@ nav: - PJM Historical Load: sdk/code-reference/pipelines/sources/spark/iso/pjm_historical_load_iso.md - CAISO Daily Load: sdk/code-reference/pipelines/sources/spark/iso/caiso_daily_load_iso.md - CAISO Historical Load: sdk/code-reference/pipelines/sources/spark/iso/caiso_historical_load_iso.md + - ERCOT Daily Load: sdk/code-reference/pipelines/sources/spark/iso/ercot_daily_load_iso.md - Weather: - Base Weather: sdk/code-reference/pipelines/sources/spark/the_weather_company/base_weather.md - Weather Forecast API V1: sdk/code-reference/pipelines/sources/spark/the_weather_company/weather_forecast_api_v1.md @@ -243,10 +244,10 @@ nav: - Queries: - Functions: - Metadata: sdk/code-reference/query/functions/metadata.md - - Time Series: + - Time Series: - Time Series Query Builder: sdk/code-reference/query/functions/time_series/time_series_query_builder.md - Raw: sdk/code-reference/query/functions/time_series/raw.md - - Latest: sdk/code-reference/query/functions/time_series/latest.md + - Latest: sdk/code-reference/query/functions/time_series/latest.md - Resample: sdk/code-reference/query/functions/time_series/resample.md - Interpolate: sdk/code-reference/query/functions/time_series/interpolate.md - Interpolation at Time: sdk/code-reference/query/functions/time_series/interpolation-at-time.md diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/iso.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/iso.py index 891402f59..0f7ec6b9e 100644 --- a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/iso.py +++ b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/iso.py @@ -65,6 +65,23 @@ ] ) +ERCOT_SCHEMA = StructType( + [ + StructField("Date", TimestampType(), True), + StructField("HourEnding", StringType(), True), + StructField("Coast", DoubleType(), True), + StructField("East", DoubleType(), True), + StructField("FarWest", DoubleType(), True), + StructField("North", DoubleType(), True), + StructField("NorthCentral", DoubleType(), True), + StructField("SouthCentral", DoubleType(), True), + StructField("Southern", DoubleType(), True), + StructField("West", DoubleType(), True), + StructField("SystemTotal", DoubleType(), True), + StructField("DstFlag", StringType(), True), + ] +) + def melt( df: DataFrame, diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/temp_cert_files.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/temp_cert_files.py new file mode 100644 index 000000000..303cbb407 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/temp_cert_files.py @@ -0,0 +1,27 @@ +import os +from tempfile import NamedTemporaryFile + + +class TempCertFiles(object): + """ + Allows to generate temporary certificate files and makes the files available for requests module to use. + """ + + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def __enter__(self): + with NamedTemporaryFile(mode="wb", delete=False) as cert_file: + cert_file.write(self.cert) + self.cert_file_name = cert_file.name + + with NamedTemporaryFile(mode="wb", delete=False) as key_file: + key_file.write(self.key) + self.key_file_name = key_file.name + + return self.cert_file_name, self.key_file_name + + def __exit__(self, exc_type, exc_value, traceback): + os.remove(self.cert_file_name) + os.remove(self.key_file_name) diff --git a/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/__init__.py b/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/__init__.py index b61ef5f44..1716c1ca7 100644 --- a/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/__init__.py +++ b/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/__init__.py @@ -1,4 +1,5 @@ from .base_iso import BaseISOSource +from .ercot_daily_load_iso import ERCOTDailyLoadISOSource from .miso_daily_load_iso import MISODailyLoadISOSource from .miso_historical_load_iso import MISOHistoricalLoadISOSource from .pjm_daily_load_iso import PJMDailyLoadISOSource diff --git a/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/base_iso.py b/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/base_iso.py index 7ad9ab02e..a1345286e 100644 --- a/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/base_iso.py +++ b/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/base_iso.py @@ -13,16 +13,15 @@ # limitations under the License. import logging -import pandas as pd -from py4j.protocol import Py4JJavaError -from pyspark.sql import DataFrame, SparkSession -import requests from datetime import datetime, timezone -import pytz +from io import BytesIO +import pandas as pd +import pytz +import requests +from pyspark.sql import DataFrame, SparkSession from pyspark.sql.types import StructType, StructField, IntegerType from requests import HTTPError -from io import BytesIO from ...interfaces import SourceInterface from ...._pipeline_utils.models import Libraries, SystemType diff --git a/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/ercot_daily_load_iso.py b/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/ercot_daily_load_iso.py new file mode 100644 index 000000000..dc3dffe7e --- /dev/null +++ b/src/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/ercot_daily_load_iso.py @@ -0,0 +1,218 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import logging +from _datetime import datetime, timedelta +from io import BytesIO +from typing import List +from zipfile import ZipFile + +import pandas as pd +import requests +from bs4 import BeautifulSoup +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 +from pyspark.sql import SparkSession +from requests import HTTPError + +from . import BaseISOSource +from ...._pipeline_utils.iso import ERCOT_SCHEMA +from ...._pipeline_utils.temp_cert_files import TempCertFiles + + +class ERCOTDailyLoadISOSource(BaseISOSource): + """ + The ERCOT Daily Load ISO Source is used to read daily load data from ERCOT using WebScrapping. + It supports actual and forecast data. +
API: https://mis.ercot.com + + + Parameters: + spark (SparkSession): Spark Session instance + options (dict): A dictionary of ISO Source specific configurations (See Attributes table below) + + Attributes: + load_type (list): Must be one of `actual` or `forecast`. + date (str): Must be in `YYYY-MM-DD` format. + certificate_pfx_key (str): The certificate key data or password received from ERCOT. + certificate_pfx_key_contents (str): The certificate data received from ERCOT, it could be base64 encoded. + + Please check the BaseISOSource for available methods. + + BaseISOSource: + ::: src.sdk.python.rtdip_sdk.pipelines.sources.spark.iso.base_iso + """ + + spark: SparkSession + options: dict + url_forecast: str = "https://mis.ercot.com/misapp/GetReports.do?reportTypeId=12312" + url_actual: str = "https://mis.ercot.com/misapp/GetReports.do?reportTypeId=13101" + url_prefix: str = "https://mis.ercot.com" + query_datetime_format: str = "%Y-%m-%d" + required_options = [ + "load_type", + "date", + "certificate_pfx_key", + "certificate_pfx_key_contents", + ] + spark_schema = ERCOT_SCHEMA + default_query_timezone = "UTC" + + def __init__(self, spark: SparkSession, options: dict) -> None: + super().__init__(spark, options) + self.spark = spark + self.options = options + self.load_type = self.options.get("load_type", "actual") + self.date = self.options.get("date", "").strip() + self.certificate_pfx_key = self.options.get("certificate_pfx_key", "").strip() + self.certificate_pfx_key_contents = self.options.get( + "certificate_pfx_key_contents", "" + ).strip() + + def generate_temp_client_cert_files_from_pfx(self): + password = self.certificate_pfx_key.encode() + pfx: bytes = base64.b64decode(self.certificate_pfx_key_contents) + + if base64.b64encode(pfx) != self.certificate_pfx_key_contents.encode(): + pfx = self.certificate_pfx_key_contents + + key, cert, _ = pkcs12.load_key_and_certificates(data=pfx, password=password) + key_bytes = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + + cert_bytes = cert.public_bytes(encoding=serialization.Encoding.PEM) + return TempCertFiles(cert_bytes, key_bytes) + + def _pull_data(self) -> pd.DataFrame: + """ + Pulls data from the ERCOT API and parses the zip files for CSV data. + + Returns: + Raw form of data. + """ + + logging.info(f"Getting {self.load_type} data for date {self.date}") + url = self.url_forecast + req_date = datetime.strptime(self.date, self.query_datetime_format) + + if self.load_type == "actual": + req_date = req_date + timedelta(days=1) + url = self.url_actual + + url_lists, files = self.generate_urls_for_zip(url, req_date) + dfs = [] + logging.info(f"Generated {len(url_lists)} URLs - {url_lists}") + logging.info(f"Requesting files - {files}") + + for url in url_lists: + df = self.download_zip(url) + dfs.append(df) + final_df = pd.concat(dfs) + return final_df + + def download_zip(self, url) -> pd.DataFrame: + logging.info(f"Downloading zip using {url}") + with self.generate_temp_client_cert_files_from_pfx() as cert: + response = requests.get(url, cert=cert) + + if not response.content: + raise HTTPError("Empty Response was returned") + + logging.info("Unzipping the file") + zf = ZipFile(BytesIO(response.content)) + csvs = [s for s in zf.namelist() if ".csv" in s] + + if len(csvs) == 0: + raise ValueError("No data was found in the specified interval") + + df = pd.read_csv(zf.open(csvs[0])) + return df + + def generate_urls_for_zip(self, url: str, date: datetime) -> (List[str], List[str]): + logging.info(f"Finding urls list for date {date}") + with self.generate_temp_client_cert_files_from_pfx() as cert: + page_response = requests.get(url, timeout=5, cert=cert) + + page_content = BeautifulSoup(page_response.content, "html.parser") + zip_info = [] + length = len(page_content.find_all("td", {"class": "labelOptional_ind"})) + + for i in range(0, length): + zip_name = page_content.find_all("td", {"class": "labelOptional_ind"})[ + i + ].text + zip_link = page_content.find_all("a")[i].get("href") + zip_info.append((zip_name, zip_link)) + + date_str = date.strftime("%Y%m%d") + zip_info = list( + filter( + lambda f_info: f_info[0].endswith("csv.zip") and date_str in f_info[0], + zip_info, + ) + ) + + urls = [] + files = [] + + if len(zip_info) == 0: + raise ValueError(f"No file was found for date - {date_str}") + + # As Forecast is generated every hour, pick the latest one. + zip_info = sorted(zip_info, key=lambda item: item[0], reverse=True) + zip_info_item = zip_info[0] + + file_name, file_url = zip_info_item + urls.append(self.url_prefix + file_url) + files.append(file_name) + + return urls, files + + def _prepare_data(self, df: pd.DataFrame) -> pd.DataFrame: + if self.load_type == "actual": + df["Date"] = pd.to_datetime(df["OperDay"], format="%m/%d/%Y") + + df = df.rename( + columns={ + "COAST": "Coast", + "EAST": "East", + "FAR_WEST": "FarWest", + "NORTH": "North", + "NORTH_C": "NorthCentral", + "SOUTH_C": "SouthCentral", + "SOUTHERN": "Southern", + "WEST": "West", + "TOTAL": "SystemTotal", + "DSTFlag": "DstFlag", + } + ) + + else: + df = df.rename(columns={"DSTFlag": "DstFlag"}) + + df["Date"] = pd.to_datetime(df["DeliveryDate"], format="%m/%d/%Y") + + return df + + def _validate_options(self) -> bool: + try: + datetime.strptime(self.date, self.query_datetime_format) + except ValueError: + raise ValueError( + f"Unable to parse date. Please specify in {self.query_datetime_format} format." + ) + return True diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_expected.csv b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_expected.csv new file mode 100644 index 000000000..84091124e --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_expected.csv @@ -0,0 +1,25 @@ +Date,HourEnding,Coast,East,FarWest,North,NorthCentral,SouthCentral,Southern,West,SystemTotal,DstFlag +2023-11-17T00:00:00,01:00,10135.49,1336.0,5757.36,989.29,10279.03,6031.93,2880.14,1238.34,38647.6,N +2023-11-17T00:00:00,02:00,9909.41,1293.77,5710.46,955.6,9907.13,5816.07,2794.81,1223.4,37610.65,N +2023-11-17T00:00:00,03:00,9771.6,1262.84,5715.07,953.4,9715.77,5733.25,2729.48,1184.61,37066.03,N +2023-11-17T00:00:00,04:00,9766.79,1261.93,5717.74,1016.36,9655.56,5707.14,2689.63,1195.55,37010.7,N +2023-11-17T00:00:00,05:00,9915.49,1266.05,5701.25,1080.7,9809.29,5825.19,2779.05,1204.86,37581.87,N +2023-11-17T00:00:00,06:00,10344.26,1351.29,5725.32,1036.34,10405.41,6122.29,2885.79,1269.09,39139.79,N +2023-11-17T00:00:00,07:00,10957.81,1463.42,5749.53,1113.97,11399.31,6698.35,3099.21,1329.38,41810.99,N +2023-11-17T00:00:00,08:00,11183.16,1489.56,5778.87,1143.22,11972.83,6973.01,3167.29,1345.47,43053.4,N +2023-11-17T00:00:00,09:00,11340.07,1508.57,6099.96,1282.17,12170.81,7041.57,3207.91,1336.44,43987.49,N +2023-11-17T00:00:00,10:00,11561.22,1342.13,6213.73,1315.28,12532.28,7008.29,3262.59,1378.89,44614.4,N +2023-11-17T00:00:00,11:00,11780.2,1468.34,6055.92,1310.07,12490.92,6890.16,3260.92,1395.85,44652.37,N +2023-11-17T00:00:00,12:00,12043.4,1581.1,6037.42,1248.41,12471.1,6820.85,3438.22,1348.85,44989.35,N +2023-11-17T00:00:00,13:00,12273.49,1611.23,6080.47,1285.75,12439.01,6855.05,3600.8,1316.79,45462.59,N +2023-11-17T00:00:00,14:00,12644.82,1651.5,6091.49,1287.25,12396.43,7079.92,3808.21,1299.69,46259.31,N +2023-11-17T00:00:00,15:00,12878.62,1626.31,6120.06,1288.4,12317.83,7351.21,3990.87,1316.69,46889.99,N +2023-11-17T00:00:00,16:00,13068.31,1616.88,6155.71,1262.62,12282.92,7631.46,4150.48,1346.38,47514.75,N +2023-11-17T00:00:00,17:00,13039.47,1643.57,6107.02,1238.44,12309.79,7739.6,4000.13,1290.92,47368.94,N +2023-11-17T00:00:00,18:00,12789.54,1670.93,5990.18,1240.34,12653.03,7787.84,3890.75,1269.9,47292.5,N +2023-11-17T00:00:00,19:00,12643.52,1667.72,6133.1,1279.62,12730.57,7832.6,3859.42,1327.01,47473.55,N +2023-11-17T00:00:00,20:00,12286.56,1634.69,6186.14,1275.64,12420.21,7647.22,3676.05,1348.56,46475.08,N +2023-11-17T00:00:00,21:00,11991.18,1576.42,6232.16,1272.85,12179.04,7385.18,3476.33,1274.84,45387.99,N +2023-11-17T00:00:00,22:00,11699.47,1534.36,6193.88,1254.31,11774.59,7141.54,3485.02,1256.44,44339.61,N +2023-11-17T00:00:00,23:00,11283.57,1451.22,6114.48,1216.52,11260.09,6778.77,3365.74,1218.28,42688.68,N +2023-11-17T00:00:00,24:00,10822.67,1382.91,6054.4,1176.03,10624.14,6362.47,3221.17,1139.7,40783.49,N diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_sample1.zip b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_actual_sample1.zip new file mode 100644 index 0000000000000000000000000000000000000000..4c94e5c987d4ebc4d71a77a67d6d416fbe5e7933 GIT binary patch literal 1330 zcmWIWW@Zs#;Nak3aL_9XXFvk(Kz4FUk)DBpfuXUXfgy;20`!awjEoHp4K4HxOih6@ zdXCN^p^iSmk-AH#bv3Xr@La!1@1hLZn>jh>V1><+lc}w*T7c`2Ds2KRliP?aTA~_vQEh{q=kM`nP{i zA6_0^{p9%Z_Uq^O+dMk{`Sk0zFKg=R?%VDCQ&U}4Q)_4cA>sYby?5m5vgGaV+Wq_Z z^fmvg_gn7GoMC=Z-gQ>juNNCqz3Q3Am2om9`ul~#1sg-UFYFF>2w$%d zx^t%I6#Gq)O^cpb-9E5O&qn@;+s)F{6dAWK8**#@K8U@+H!H~ zZ&nf24UR$cOAnq?Ug}=r?O{1ZVR6BYUT)8oAx>;x8>ThOs?E5k6#Xc7_0ORAm>urx zoA)K~tXMK}!HovF)%+5USAREUS?u1`9DQ(m=INLMj;i8}2b){{K5Sa|qIj{h#QUyW z%uAZ?1e)Glp5+=>^*q>+BP67AGW4+3~?_ z_O+>fmxaupv*}LXHYp(Q&dLskU$a~vzPk|0_p2c=VsZP$^O*@tHoOtIt#+YZy58cf zmDq(qw#ss|ol!1Xf7X}TGG1*m^*QmIFNW*Q_8(@pwvjpynHRjzzY%0MmCyd1{=38{QAmCzfL)UeG6LpvCa}P=sYV3CJZ=Uzzx64}2{ruPVeQDv2G7Ndf{pz6c z1! z^(#y7i*6IMc9t$Xw*Ae5_J;C^ZCvc%ENtVtimq&b)Ks@A>u3*?s-P5~htPsAmO;lX zlRGuIDvvXHxlHSJt~x(c)nR*T$Q!xZzst3<)ZTMc9V|GZz`J5khnexyL(6KMY$CR* zT{?Q}ZdY8F(^r`z2@dxX?vj^DN-)B4u2zP8tvRJXJ>X=`c>Giba;qJ9XydDxVnBlZ{@{+RJaut5{Ka|FBl3qv9PO z%So37TP&_~teoS&Pjo0gfAs#lSl6B@$Hz|O=GoBEJFHWfsd zR&X;gvV3J^U|=^n|2@557KOANa#M0vebmNHD7lNHZ473N-(amgr#WQ&e;eQ1dv>q#))H ze5gZUy4Sh$SG{#KPIz8D?Wv=usq5$K>8axx%E!T``RBOYY_3JR?^d5`$N>AGAc$pW z4A6}rcLsPfGI25E4j!0Sw={xS*u8->uo#&{m=VE)Y%M5wU|>sQG>}PdUXnpT9y&S0|3Z~W$pj~ literal 0 HcmV?d00001 diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_expected.csv b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_expected.csv new file mode 100644 index 000000000..b556b3a13 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_expected.csv @@ -0,0 +1,193 @@ +Date,HourEnding,Coast,East,FarWest,North,NorthCentral,SouthCentral,Southern,West,SystemTotal,DstFlag +2023-11-19T00:00:00,1:00,10162.2002,1255.85,5951.04,1125.3101,9868.4404,5857.4502,3016.8,1137.24,38374.3309,N +2023-11-19T00:00:00,2:00,9884.8203,1260.22,6184.75,1111.24,9562.4199,5709.6299,2951.95,1090.33,37755.3601,N +2023-11-19T00:00:00,3:00,9679.6699,1251.61,6095.9302,1042.3101,9388.0195,5551.0601,2793.3,1017.26,36819.1598,N +2023-11-19T00:00:00,4:00,9573.21,1226.13,6179.3901,1051.3199,9225.8604,5439.9199,2750.0701,862.481,36308.3814,N +2023-11-19T00:00:00,5:00,9527.0498,1227.5699,6288.9502,1088.46,9199.1104,5389.6099,2653.3101,692.937,36066.9973,N +2023-11-19T00:00:00,6:00,9602.21,1285.51,6309.6699,1126.6899,9372.3203,5425.0298,2728.3501,777.816,36627.596,N +2023-11-19T00:00:00,7:00,9739.4102,1342.78,6187.2202,1125.4301,9605.96,5676.1699,2769.1799,994.439,37440.5893,N +2023-11-19T00:00:00,8:00,9809.0303,1374.53,6020.54,1114.61,9860.9697,5796.0,2793.1499,1316.77,38085.5999,N +2023-11-19T00:00:00,9:00,10288.4004,1414.0601,6072.71,1168.5601,10442.0,6136.4399,2975.78,1491.97,39989.9205,N +2023-11-19T00:00:00,10:00,10697.4004,1426.96,6126.8999,1168.83,11055.0996,6435.8999,3143.4299,1547.14,41601.6597,N +2023-11-19T00:00:00,11:00,10939.5996,1445.11,6166.2202,1149.87,11431.7998,6616.6802,3301.5901,1364.46,42415.3299,N +2023-11-19T00:00:00,12:00,11091.7998,1429.73,6159.3901,1116.6801,11614.2998,6750.3701,3468.7,1315.6899,42946.6598,N +2023-11-19T00:00:00,13:00,11295.2002,1433.23,6167.1899,1125.72,11700.2998,6873.8198,3689.72,1290.9301,43576.1098,N +2023-11-19T00:00:00,14:00,11366.7998,1458.96,6166.0498,1181.54,11735.5,6904.0098,3708.1201,1309.85,43830.8295,N +2023-11-19T00:00:00,15:00,11510.2002,1461.92,6171.4902,1207.65,11785.4004,6894.3901,3652.05,1316.0699,43999.1708,N +2023-11-19T00:00:00,16:00,11513.2002,1470.29,6204.2002,1198.23,11670.5996,6863.1699,3730.6899,1283.78,43934.1598,N +2023-11-19T00:00:00,17:00,11523.5,1492.03,6034.1299,1217.74,11791.2002,6918.0098,3757.4399,1305.64,44039.6898,N +2023-11-19T00:00:00,18:00,11825.5996,1554.45,6016.6001,1209.99,12190.0996,7102.04,3806.97,1357.8101,45063.5594,N +2023-11-19T00:00:00,19:00,12184.4004,1640.74,6142.1299,1258.97,12379.5996,7411.9399,3959.3,1365.83,46342.9098,N +2023-11-19T00:00:00,20:00,12125.7002,1590.4399,6398.2402,1236.78,12172.5,7404.1001,3938.75,1209.0699,46075.5803,N +2023-11-19T00:00:00,21:00,11874.0,1552.54,6475.4902,1218.58,11660.2002,7258.8501,3817.3101,1125.84,44982.8106,N +2023-11-19T00:00:00,22:00,11591.9004,1458.36,6475.2998,1167.48,11238.0,6935.1899,3606.5701,1090.09,43562.8902,N +2023-11-19T00:00:00,23:00,11299.0996,1417.79,6284.7402,1090.9399,10707.7002,6636.96,3315.3501,968.139,41720.719,N +2023-11-19T00:00:00,24:00,10881.0996,1321.9301,6258.48,1037.1,10171.2998,6271.98,3139.02,937.574,40018.4835,N +2023-11-20T00:00:00,1:00,10480.9004,1251.7,6247.6899,991.738,9650.5,5947.1299,2978.5901,911.51,38459.7583,N +2023-11-20T00:00:00,2:00,10251.5996,1223.0699,6105.6602,968.97,9380.4404,5713.3398,2900.27,934.034,37477.3839,N +2023-11-20T00:00:00,3:00,10162.9004,1215.4399,5931.0298,969.206,9335.75,5595.3101,2858.8899,950.452,37018.9781,N +2023-11-20T00:00:00,4:00,10222.5996,1219.63,5772.5498,1011.11,9392.6201,5584.8301,2855.27,966.757,37025.3666,N +2023-11-20T00:00:00,5:00,10361.0996,1228.98,5704.3301,1068.9301,9633.25,5665.3301,2910.52,991.071,37563.5109,N +2023-11-20T00:00:00,6:00,10616.4004,1313.45,5719.3101,1098.9,10300.9004,5974.23,2981.1699,1051.87,39056.2308,N +2023-11-20T00:00:00,7:00,10949.9004,1419.09,5995.96,1186.05,11358.2998,6533.5298,3133.55,1203.58,41779.96,N +2023-11-20T00:00:00,8:00,11123.2998,1463.8199,6042.0898,1252.0601,11815.7998,6807.9399,3192.8,1281.75,42979.5593,N +2023-11-20T00:00:00,9:00,11340.7998,1492.28,5996.8701,1347.92,12007.2002,6817.1899,3231.6799,1318.78,43552.7199,N +2023-11-20T00:00:00,10:00,11827.9004,1412.9,6112.73,1351.9301,12288.5,6845.5898,3399.29,1364.53,44603.3703,N +2023-11-20T00:00:00,11:00,12260.9004,1511.65,6139.6499,1350.41,12446.0996,6809.7598,3543.9099,1371.42,45433.7996,N +2023-11-20T00:00:00,12:00,12726.4004,1608.39,6071.9302,1315.71,12617.4004,6832.3198,3820.1699,1391.3,46383.6207,N +2023-11-20T00:00:00,13:00,13111.9004,1656.38,6150.7202,1349.75,12692.2998,6939.1602,4042.9199,1373.99,47317.1205,N +2023-11-20T00:00:00,14:00,13458.2998,1681.24,6215.0898,1366.4399,12750.0996,7064.3999,4298.3101,1381.97,48215.8491,N +2023-11-20T00:00:00,15:00,13645.2998,1665.64,6110.9102,1365.6801,12777.2002,7303.0801,4470.46,1399.38,48737.6504,N +2023-11-20T00:00:00,16:00,13729.5,1661.65,5924.2598,1295.04,12763.9004,7563.0298,4534.0498,1406.42,48877.8498,N +2023-11-20T00:00:00,17:00,13631.7002,1681.71,5804.4199,1242.41,12833.7002,7694.48,4362.6201,1382.64,48633.6804,N +2023-11-20T00:00:00,18:00,13463.2002,1699.8199,5745.52,1288.64,13251.0,7816.1899,4192.3301,1332.42,48789.1201,N +2023-11-20T00:00:00,19:00,13266.2998,1694.58,5837.3398,1404.7,13365.9004,7892.73,4139.3999,1371.42,48972.3699,N +2023-11-20T00:00:00,20:00,12873.5,1680.7,5847.23,1424.99,12954.2998,7645.8901,3955.01,1372.0,47753.6199,N +2023-11-20T00:00:00,21:00,12635.5996,1654.39,5938.8101,1421.61,12598.7998,7368.79,3773.78,1346.58,46738.3595,N +2023-11-20T00:00:00,22:00,12384.7002,1593.38,6168.96,1396.29,12084.2998,7009.98,3718.23,1292.05,45647.89,N +2023-11-20T00:00:00,23:00,11729.7002,1491.42,6178.0098,1317.36,11342.7998,6570.0801,3592.1699,1217.5,43439.0398,N +2023-11-20T00:00:00,24:00,10995.0,1395.35,6127.6499,1211.3,10565.5996,6118.6001,3396.4399,1134.26,40944.1995,N +2023-11-21T00:00:00,1:00,10335.2998,1282.6899,6140.71,1102.89,9783.5801,6054.5898,3138.0,1107.53,38945.2896,N +2023-11-21T00:00:00,2:00,10014.5,1258.59,6201.9102,1070.12,9644.5703,5896.8301,3025.53,1117.5,38229.5506,N +2023-11-21T00:00:00,3:00,9790.8701,1250.27,6100.8101,1063.97,9600.8604,5802.0601,2933.4099,1107.22,37649.4706,N +2023-11-21T00:00:00,4:00,9784.2197,1260.29,6092.29,1065.5699,9615.3301,5756.75,2892.8101,1101.33,37568.5898,N +2023-11-21T00:00:00,5:00,9980.5,1300.01,6138.1602,1096.85,9901.6699,5842.7998,2894.1699,1108.52,38262.6798,N +2023-11-21T00:00:00,6:00,10553.4004,1386.95,6145.3901,1160.27,10598.0996,6169.5498,2973.1101,1154.78,40141.55,N +2023-11-21T00:00:00,7:00,11371.4004,1538.9,6296.4902,1271.13,11669.0,6750.73,3143.95,1253.5,43295.1006,N +2023-11-21T00:00:00,8:00,11810.0996,1614.11,6197.6699,1327.0699,12328.5,6992.7998,3218.0701,1303.64,44791.9593,N +2023-11-21T00:00:00,9:00,12057.7002,1646.1801,5979.6099,1341.24,12474.0996,6992.7202,3227.0601,1331.77,45050.3801,N +2023-11-21T00:00:00,10:00,12151.0,1662.1,5905.23,1334.14,12650.7998,6991.3599,3271.1899,1366.97,45332.7896,N +2023-11-21T00:00:00,11:00,12232.9004,1670.04,5900.4199,1335.08,12599.0,6966.9399,3293.52,1388.8,45386.7002,N +2023-11-21T00:00:00,12:00,12300.5996,1659.39,5998.98,1322.9,12591.2998,6928.5498,3260.48,1389.72,45451.9192,N +2023-11-21T00:00:00,13:00,12346.4004,1661.61,6073.6401,1312.64,12550.9004,6895.3999,3209.04,1349.1,45398.7308,N +2023-11-21T00:00:00,14:00,12370.5996,1665.47,6015.52,1303.73,12476.2998,6890.8599,3164.6699,1326.66,45213.8092,N +2023-11-21T00:00:00,15:00,12414.5996,1657.87,5877.6699,1298.86,12350.0,6877.73,3113.01,1317.61,44907.3495,N +2023-11-21T00:00:00,16:00,12349.7998,1659.61,5778.98,1296.8199,12296.5996,6896.1401,3058.8101,1308.6801,44645.4396,N +2023-11-21T00:00:00,17:00,12344.7998,1670.5699,5751.3301,1315.6801,12420.5,7042.6099,3049.78,1307.12,44902.3898,N +2023-11-21T00:00:00,18:00,12451.7002,1715.41,5758.5601,1371.77,12861.0,7265.8999,3080.3799,1321.24,45825.9601,N +2023-11-21T00:00:00,19:00,12548.5,1772.97,6005.4502,1440.35,13284.7002,7647.1602,3197.1799,1380.39,47276.7005,N +2023-11-21T00:00:00,20:00,12552.5,1768.87,6180.2598,1440.78,13275.9004,7683.6899,3214.51,1413.64,47530.1501,N +2023-11-21T00:00:00,21:00,12426.5,1750.48,6254.1201,1438.46,13133.5,7633.2202,3196.6599,1422.5,47255.4402,N +2023-11-21T00:00:00,22:00,12167.7002,1702.52,6383.4302,1410.35,12760.2998,7440.7402,3143.05,1399.01,46407.1004,N +2023-11-21T00:00:00,23:00,11626.2998,1623.3,6263.7998,1349.04,12128.0996,7077.9399,3003.02,1316.0,44387.4991,N +2023-11-21T00:00:00,24:00,11076.5,1548.46,6249.6001,1296.99,11553.5996,6743.25,2850.3899,1250.09,42568.8796,N +2023-11-22T00:00:00,1:00,10088.0996,1493.25,6307.3999,1237.62,11432.5,6534.3599,2797.3701,1156.71,41047.3095,N +2023-11-22T00:00:00,2:00,9854.2695,1451.64,6279.0498,1198.36,11124.7998,6312.9902,2743.76,1106.72,40071.5893,N +2023-11-22T00:00:00,3:00,9724.71,1438.45,6221.73,1188.95,11010.5996,6196.0,2709.54,1082.6,39572.5796,N +2023-11-22T00:00:00,4:00,9742.25,1453.27,6130.79,1201.86,11102.9004,6193.0601,2702.1001,1097.08,39623.3106,N +2023-11-22T00:00:00,5:00,9948.3896,1510.29,6111.0601,1240.72,11519.0996,6350.8501,2741.0,1128.99,40550.3994,N +2023-11-22T00:00:00,6:00,10484.4004,1646.25,6106.2998,1334.14,12537.5,6765.5801,2859.53,1216.46,42950.1603,N +2023-11-22T00:00:00,7:00,11215.7002,1868.02,6227.6602,1499.73,14066.5996,7467.4502,3077.25,1379.26,46801.6702,N +2023-11-22T00:00:00,8:00,11622.7002,1963.6,6202.0098,1576.46,14793.0,7853.79,3181.47,1457.78,48650.81,N +2023-11-22T00:00:00,9:00,11827.9004,1974.87,6156.0498,1558.63,14651.4004,7871.1001,3201.52,1436.96,48678.4307,N +2023-11-22T00:00:00,10:00,11910.2998,1936.86,6175.79,1530.53,14354.5,7821.7402,3235.6399,1353.89,48319.2499,N +2023-11-22T00:00:00,11:00,11923.2002,1880.27,6109.3701,1462.33,13977.2002,7723.04,3255.3101,1303.0601,47633.7807,N +2023-11-22T00:00:00,12:00,11829.7998,1836.55,6025.52,1374.59,13552.2998,7600.8301,3246.0,1256.8,46722.3897,N +2023-11-22T00:00:00,13:00,11693.5,1803.1,6018.3198,1320.62,13204.5,7495.8101,3218.73,1199.21,45953.7899,N +2023-11-22T00:00:00,14:00,11603.0,1794.03,6025.4199,1286.0699,12923.5996,7443.52,3210.5,1155.65,45441.7894,N +2023-11-22T00:00:00,15:00,11491.0996,1774.9399,6009.1201,1252.78,12733.5,7407.3999,3200.6201,1142.9399,45012.3995,N +2023-11-22T00:00:00,16:00,11371.5996,1778.42,5937.4102,1244.98,12748.9004,7458.0,3184.6599,1130.39,44854.3601,N +2023-11-22T00:00:00,17:00,11309.9004,1813.98,5856.1299,1269.87,13043.5996,7642.4502,3225.5,1124.12,45285.5501,N +2023-11-22T00:00:00,18:00,11548.9004,1896.5601,5759.8599,1348.4,13856.4004,7968.96,3303.05,1183.08,46865.2108,N +2023-11-22T00:00:00,19:00,11795.5,1991.46,5810.0498,1440.71,14521.0,8336.8398,3454.6101,1295.3101,48645.4798,N +2023-11-22T00:00:00,20:00,11816.2998,1994.71,5839.8701,1434.76,14489.5996,8343.4199,3493.48,1338.35,48750.4894,N +2023-11-22T00:00:00,21:00,11711.5,1979.77,5875.3501,1418.4301,14378.5996,8251.9697,3498.76,1366.64,48481.0195,N +2023-11-22T00:00:00,22:00,11451.5996,1909.91,6085.1499,1394.66,13946.2998,8027.4399,3454.52,1351.2,47620.7792,N +2023-11-22T00:00:00,23:00,11033.5996,1784.4301,6181.8901,1347.0,13348.9004,7646.4502,3349.0901,1311.34,46002.7005,N +2023-11-22T00:00:00,24:00,10633.0996,1659.33,6224.3101,1302.52,12780.4004,7261.2598,3220.3899,1298.1801,44379.4899,N +2023-11-23T00:00:00,1:00,10201.7002,1555.22,6327.71,1280.9,11487.2998,6821.27,3114.6001,1318.86,42107.5601,N +2023-11-23T00:00:00,2:00,9942.8701,1499.25,6454.5298,1242.5,10981.5996,6571.9399,2980.6799,1324.92,40998.2893,N +2023-11-23T00:00:00,3:00,9750.2695,1476.8199,6180.4702,1246.92,10660.2998,6402.96,2871.96,1322.46,39912.1594,N +2023-11-23T00:00:00,4:00,9634.54,1463.79,5809.6802,1191.8199,10283.4004,6301.2798,2835.3301,1327.63,38847.4704,N +2023-11-23T00:00:00,5:00,9721.1699,1436.48,5840.5698,1179.51,10199.4004,6299.96,2874.75,1319.04,38870.8801,N +2023-11-23T00:00:00,6:00,10076.0,1451.97,5973.46,1261.29,10359.0996,6408.1699,3012.26,1361.5,39903.7495,N +2023-11-23T00:00:00,7:00,10574.0,1500.0601,6138.73,1351.42,10728.0996,6588.73,3252.6101,1425.47,41559.1198,N +2023-11-23T00:00:00,8:00,10970.2998,1567.78,5703.3701,1429.24,11407.0,6780.8301,3417.52,1484.79,42760.83,N +2023-11-23T00:00:00,9:00,11731.5996,1648.6899,5512.5698,1465.51,12254.0996,7142.0,3606.02,1542.0601,44902.549,N +2023-11-23T00:00:00,10:00,12492.7002,1694.4,5393.8501,1491.25,13123.5,7528.0098,3781.54,1530.5,47035.7501,N +2023-11-23T00:00:00,11:00,13103.9004,1717.63,5326.7002,1477.49,13503.7998,7775.98,3821.2,1502.4,48229.1004,N +2023-11-23T00:00:00,12:00,13348.5,1689.6801,5573.1699,1413.8101,13642.5,7851.5,3790.0701,1430.85,48740.0802,N +2023-11-23T00:00:00,13:00,13559.5996,1615.86,5816.46,1349.39,13476.2002,7755.5801,3663.1499,1322.1,48558.3398,N +2023-11-23T00:00:00,14:00,13573.5,1536.73,5774.3501,1271.47,12997.9004,7601.27,3440.51,1209.64,47405.3705,N +2023-11-23T00:00:00,15:00,13616.0996,1479.14,5668.0498,1215.37,12478.7002,7517.1699,3272.22,1173.55,46420.2995,N +2023-11-23T00:00:00,16:00,13531.7002,1443.03,5686.0801,1172.87,12148.5,7464.02,3218.28,1096.88,45761.3603,N +2023-11-23T00:00:00,17:00,13631.7998,1451.9399,5551.1899,1145.25,11947.5996,7480.1802,3232.97,1069.1,45510.0294,N +2023-11-23T00:00:00,18:00,13897.2998,1531.11,5575.6201,1182.4399,12289.4004,7615.6401,3309.8999,1152.45,46553.8602,N +2023-11-23T00:00:00,19:00,13640.9004,1569.1,5814.98,1263.17,12460.5,7722.8501,3377.1201,1198.46,47047.0806,N +2023-11-23T00:00:00,20:00,13343.7998,1572.1,5858.52,1296.9301,12487.5996,7698.7202,3254.6299,1222.3,46734.5996,N +2023-11-23T00:00:00,21:00,13075.7002,1578.54,5836.6899,1304.4,12508.0,7615.98,3181.7,1271.9399,46372.95,N +2023-11-23T00:00:00,22:00,12758.5,1548.8,5999.6499,1317.34,12492.0996,7503.23,3120.46,1277.25,46017.3295,N +2023-11-23T00:00:00,23:00,12252.4004,1503.25,5944.1899,1313.83,12240.0,7295.8101,3010.26,1275.97,44835.7104,N +2023-11-23T00:00:00,24:00,11706.0,1460.84,5840.6602,1293.72,11849.5,7086.7998,2922.6899,1277.34,43437.5499,N +2023-11-24T00:00:00,1:00,10687.4004,1410.29,5993.8701,1278.21,11332.7998,6996.4302,2848.9299,1272.01,41819.9404,N +2023-11-24T00:00:00,2:00,10456.4004,1367.79,6184.96,1246.85,10895.7002,6840.6699,2733.5701,1285.34,41011.2806,N +2023-11-24T00:00:00,3:00,10322.2002,1397.79,6069.4199,1264.3,10624.4004,6748.6001,2617.3,1305.08,40349.0906,N +2023-11-24T00:00:00,4:00,10345.2998,1423.4399,6125.0098,1294.92,10575.5996,6728.0298,2562.5701,1306.1899,40361.0589,N +2023-11-24T00:00:00,5:00,10498.9004,1471.7,6038.5601,1315.3199,10788.7002,6867.8101,2586.05,1360.6801,40927.7208,N +2023-11-24T00:00:00,6:00,10847.7002,1583.63,6027.27,1383.38,11415.4004,7215.3901,2670.6201,1394.8199,42538.2107,N +2023-11-24T00:00:00,7:00,11380.0,1726.84,6083.04,1455.6801,12166.7002,7649.73,2880.28,1449.02,44791.2903,N +2023-11-24T00:00:00,8:00,11562.0,1864.54,5792.3999,1528.76,12848.7002,7994.48,3006.5,1606.15,46203.5301,N +2023-11-24T00:00:00,9:00,11783.4004,1901.6899,5737.7402,1559.5601,13196.2998,8028.25,3142.72,1548.37,46898.0304,N +2023-11-24T00:00:00,10:00,11831.0,1876.1801,5718.7598,1517.0601,13275.7998,7684.0,3246.8899,1437.5699,46587.2596,N +2023-11-24T00:00:00,11:00,11841.7002,1821.05,5794.4102,1483.38,12970.2998,7105.1099,3210.77,1331.04,45557.7601,N +2023-11-24T00:00:00,12:00,11770.9004,1704.95,5751.6602,1436.89,12573.2998,6614.7002,3137.1799,1286.85,44276.4305,N +2023-11-24T00:00:00,13:00,11600.2998,1606.2,5617.27,1380.85,12100.4004,6345.9102,3123.23,1289.41,43063.5704,N +2023-11-24T00:00:00,14:00,11515.0,1529.16,5406.6299,1360.98,11565.0,6236.9199,3164.76,1325.63,42104.0798,N +2023-11-24T00:00:00,15:00,11657.7002,1490.26,5472.7002,1321.0,11295.9004,6245.23,3217.27,1241.1899,41941.2507,N +2023-11-24T00:00:00,16:00,11980.2002,1521.41,5672.2202,1279.92,11251.5996,6480.0,3289.71,1238.3199,42713.3799,N +2023-11-24T00:00:00,17:00,12412.5996,1634.6,5712.54,1280.08,11639.2998,6850.0098,3449.1599,1334.48,44312.7691,N +2023-11-24T00:00:00,18:00,13098.0996,1787.13,5799.71,1346.87,12416.0996,7520.4902,3607.27,1465.97,47041.6394,N +2023-11-24T00:00:00,19:00,13232.2002,1857.4,5749.8999,1447.61,12733.7002,7867.6201,3755.3501,1616.03,48259.8105,N +2023-11-24T00:00:00,20:00,12965.5,1842.47,5827.8999,1480.09,12475.4004,7686.8101,3695.3601,1539.08,47512.6105,N +2023-11-24T00:00:00,21:00,12826.0,1837.1899,5910.73,1477.6,12245.0996,7436.2202,3670.5801,1489.4301,46892.8499,N +2023-11-24T00:00:00,22:00,12684.5996,1794.0699,6107.0098,1454.0601,11925.9004,7278.77,3655.6799,1412.17,46312.2597,N +2023-11-24T00:00:00,23:00,12172.0996,1641.77,5927.23,1385.6899,11465.5996,6904.3501,3446.8401,1387.27,44330.8493,N +2023-11-24T00:00:00,24:00,11611.0996,1559.51,5946.75,1304.01,10944.2002,6528.2402,3363.29,1309.76,42566.86,N +2023-11-25T00:00:00,1:00,10433.5,1579.39,5789.4102,1215.28,11372.9004,6429.02,3085.54,1282.0601,41187.1007,N +2023-11-25T00:00:00,2:00,10130.9004,1527.29,5734.1899,1194.9,11129.7002,6237.9102,2920.8601,1215.59,40091.3408,N +2023-11-25T00:00:00,3:00,9838.0801,1480.36,5751.6401,1242.77,11157.9004,6159.3398,2747.98,1171.97,39550.0404,N +2023-11-25T00:00:00,4:00,9774.4697,1444.47,5759.0498,1244.72,10982.5,6104.0098,2636.3701,1147.92,39093.5094,N +2023-11-25T00:00:00,5:00,9782.21,1502.86,5762.3799,1279.1899,11087.7002,6125.2002,2611.1399,1154.91,39305.5901,N +2023-11-25T00:00:00,6:00,9921.8096,1557.14,5805.1699,1308.5699,11386.9004,6280.3398,2683.3999,1172.22,40115.5495,N +2023-11-25T00:00:00,7:00,10231.7998,1637.27,5848.2798,1413.42,11947.5,6592.7402,2770.9099,1219.12,41661.0397,N +2023-11-25T00:00:00,8:00,10535.4004,1705.42,5938.7998,1443.97,12555.2998,6979.6699,2817.01,1273.53,43249.0999,N +2023-11-25T00:00:00,9:00,10894.5996,1745.87,5997.77,1388.9301,12948.7998,7210.6001,2940.27,1360.77,44487.6096,N +2023-11-25T00:00:00,10:00,11113.2002,1760.15,6070.27,1498.35,13011.7998,7235.7598,3050.8301,1459.5601,45199.92,N +2023-11-25T00:00:00,11:00,11218.5,1755.8101,6152.1401,1454.2,12808.2002,6977.02,3151.8401,1512.54,45030.2505,N +2023-11-25T00:00:00,12:00,11271.7998,1739.67,6209.3599,1460.47,12432.0,6742.1201,3248.6599,1558.97,44663.0497,N +2023-11-25T00:00:00,13:00,11368.7998,1701.9301,6174.6699,1417.91,12156.7002,6525.5098,3279.6499,1566.91,44192.0797,N +2023-11-25T00:00:00,14:00,11431.7998,1631.34,6275.0601,1362.67,11851.2002,6334.6802,3364.5601,1551.12,43802.4304,N +2023-11-25T00:00:00,15:00,11401.5,1605.23,6268.5601,1343.95,11735.7002,6269.6201,3464.29,1551.55,43640.4004,N +2023-11-25T00:00:00,16:00,11285.5,1593.51,6187.52,1338.26,11810.2998,6431.6499,3441.6201,1544.13,43632.4898,N +2023-11-25T00:00:00,17:00,11307.0,1615.51,6079.6201,1328.85,12036.0996,6707.0601,3459.8201,1533.37,44067.3299,N +2023-11-25T00:00:00,18:00,11425.5996,1685.3199,6003.5098,1321.4301,12537.0,7204.5,3423.96,1503.5601,45104.8795,N +2023-11-25T00:00:00,19:00,11585.2002,1789.41,5930.29,1380.98,12988.2998,7480.0898,3366.53,1541.8199,46062.6197,N +2023-11-25T00:00:00,20:00,11595.9004,1811.66,5863.8701,1342.15,12918.0,7215.46,3296.3501,1551.29,45594.6806,N +2023-11-25T00:00:00,21:00,11559.7998,1792.13,5864.23,1337.24,12937.0,6927.52,3291.3899,1539.37,45248.6797,N +2023-11-25T00:00:00,22:00,11432.5,1801.08,5924.8999,1297.36,12825.7002,6725.1699,3251.24,1490.0699,44748.0199,N +2023-11-25T00:00:00,23:00,11122.2002,1775.24,5946.5298,1288.59,12683.7002,6434.2002,3259.6599,1434.35,43944.4703,N +2023-11-25T00:00:00,24:00,10760.7998,1700.96,5896.1699,1237.21,12382.2002,6170.2998,3201.8301,1389.78,42739.2498,N +2023-11-26T00:00:00,1:00,9842.46,1539.0,5853.6401,1146.21,11817.2998,6039.23,3031.04,1336.6,40605.4799,N +2023-11-26T00:00:00,2:00,9548.7197,1490.97,5787.1401,1173.03,11579.0,5877.6201,2913.79,1286.54,39656.8099,N +2023-11-26T00:00:00,3:00,9296.1201,1442.63,5830.29,1188.5601,11611.4004,5762.8501,2821.9199,1248.27,39202.0406,N +2023-11-26T00:00:00,4:00,9212.46,1415.54,5842.02,1160.83,11458.7998,5722.8101,2670.1001,1226.0601,38708.6201,N +2023-11-26T00:00:00,5:00,9205.1797,1451.08,5830.02,1190.05,11613.7002,5682.7998,2597.98,1221.59,38792.3997,N +2023-11-26T00:00:00,6:00,9307.5,1472.6,5853.8999,1216.4399,11970.0,5685.6499,2677.5701,1239.0601,39422.7199,N +2023-11-26T00:00:00,7:00,9547.46,1510.35,5880.52,1330.67,12515.4004,5745.3301,2790.8501,1290.39,40610.9706,N +2023-11-26T00:00:00,8:00,9808.4902,1541.95,5951.0,1358.17,13231.2002,5920.7002,2851.8101,1353.51,42016.8307,N +2023-11-26T00:00:00,9:00,10224.7998,1578.59,5984.27,1340.01,13688.4004,6118.1499,3002.1299,1441.55,43377.9,N +2023-11-26T00:00:00,10:00,10616.5996,1622.86,6085.0,1444.0,13732.7002,6291.5698,3164.3701,1517.16,44474.2597,N +2023-11-26T00:00:00,11:00,10895.2002,1639.1,6144.1899,1452.89,13338.2998,6256.8198,3221.4199,1541.5601,44489.4797,N +2023-11-26T00:00:00,12:00,11090.0,1662.24,6185.6699,1453.34,12786.0,6230.96,3244.8301,1570.03,44223.07,N +2023-11-26T00:00:00,13:00,11347.0996,1671.41,6126.8301,1424.63,12390.5,6210.4902,3188.27,1574.77,43933.9999,N +2023-11-26T00:00:00,14:00,11535.0,1647.8101,6237.3198,1351.08,11973.2002,6145.7002,3136.3101,1554.23,43580.6504,N +2023-11-26T00:00:00,15:00,11611.0,1653.2,6248.1802,1347.9399,11785.2998,6103.9502,3075.3501,1545.36,43370.2802,N +2023-11-26T00:00:00,16:00,11566.2998,1667.14,6142.8101,1348.85,11838.7002,6199.75,2972.25,1527.67,43263.4701,N +2023-11-26T00:00:00,17:00,11580.7998,1676.38,5991.1001,1343.73,12148.0996,6417.0698,2914.1299,1503.2,43574.5092,N +2023-11-26T00:00:00,18:00,11704.0,1722.1,5921.96,1334.04,13075.5996,6859.7598,2994.4399,1474.12,45086.0193,N +2023-11-26T00:00:00,19:00,11800.5,1792.05,5860.1802,1433.17,13952.5,7135.71,3179.47,1538.03,46691.6102,N +2023-11-26T00:00:00,20:00,11759.5996,1813.25,5783.3101,1386.5699,14149.2002,7086.8901,3259.9199,1564.5699,46803.3097,N +2023-11-26T00:00:00,21:00,11619.9004,1797.29,5775.1899,1392.49,14265.7002,6959.02,3332.78,1573.48,46715.8505,N +2023-11-26T00:00:00,22:00,11340.7998,1798.59,5855.23,1333.92,14049.0996,6859.9199,3308.8401,1524.7,46071.0994,N +2023-11-26T00:00:00,23:00,10908.5,1769.85,5911.8301,1311.71,13455.2998,6619.73,3217.8799,1452.16,44646.9598,N +2023-11-26T00:00:00,24:00,10413.9004,1707.4,5887.3901,1259.4301,12623.5996,6430.0801,2977.9099,1371.99,42671.7002,N diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_sample1.zip b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/data/ercot_daily_load_forecast_sample1.zip new file mode 100644 index 0000000000000000000000000000000000000000..44883047845d11e7d29fb24f5ca5654e4f79a5fb GIT binary patch literal 8316 zcmbt)MNk|J&?S(;Ed+OWXK;dRaF@Y?5ANYpT2h5FntTp&<-(dm148FUb7It*kw`czAet`GCB9|3m&CaPjf{ zSLfvw;o<`V|5LbB6=V(MrFE6$wbeC&0)o6;R-Qi2`6|gAV~J7s*TxtJev-ClsI~`* z*wdf^G^%%PfblDYw|h^CwA2;IMdIbk#i_Ldz~9erylUt5p}qfYC%OOS%vbW|Y45B* z{PzM4x$2gDeZ&oY{>A+-_1g97P3=>8-&JVE%_M8` z6&Gu!0atIoLFn!FwHVA`Rk8;v@dxf$>+AT`-uD#LUaw*Bx*K>R>)odkxT0qlxR!f& zmTaC#kZ=xz2WB$l*_8RRHl*Hix16nc96YhVs z{J_TEf$0W`L;Jf!sw-OJMXT=(Yb(_+sAAUg^t0m3pwr*7fAay4f*PjVuzz?ZcW2t` zSKO7@nDw4A9PU2AHljiYCqM^I3XA@U2Glx%Voqz(M8f_t+czbg#3{sm&+z0Cj(fh| zpc+5YzIE8B?v4FN5g`+qkbG-P4Sxv7#>CO612o^?p+(=4{w~Dt@15o^G!PwsE*1C( zFwMu#0kuV@4EH*6)$Na{pUsw^&nkzkhuy}yaV1RU)R$bO6R-CN?>G8+N1P-}XE6d& zqo=AKHRh&A3+Q&Jp(pgW!q5F56|1)nM!&DAOcj1-(OtDCm0zc9fwGw=(o7hW$}jG7 zUyQDe6^nsO8V7iheMu_)iRsOeYk!RpDmM2)qQ&$U0s+mR=4ddh-KbN4O5;&PPD3~ zQb~WsJ3NNHnzzckdKQ#WLzWT*YR2pfKw*?gZe_b?&AOZYhzlQwwX5NTk2yb5cn*dS zJLPFUI`8cB+WMdO%&w*e)OcuKMpEIQPWnH&O;V1yX6bo zqZW0;((eIC5mGavp^qqq^XaSb=a4XO9}^Y^AnpdO-{SLB>XZ>)Ur9`+C_UU(XLhr; z)a6?k%W0%H`M}=Wp&Mfq>DHRM?BJ0YFWb@LG%-MUAfc@h>{+=8%O_UUTx~I#O54VZ z1i>MhR6HP$b8^M`0|}2`3pWdY1PkQGI>*qm^J`sFQ8Okgn5p8MM}D2_t@kS`vxK8l z3VY=*$)38Y5M}y7ni-O7;uh*inh}_?d!PAJU^?^)nVQ56zCs=)gLlJ(mQyVpiE)1G z(Ttd0vq}*Er`aSeY)^JSD|8pk&}QxA5c3J^L*{;W9!=$0i>j>i2i|oG{ZzTsX1oN7 zDxDjlk04xq8O9L@BRC%=aliIE4CO9fSey1a34z@y?~wo8!cSe4X^UnZDC)xvyAjo^q;MvAHb?s*TuYo;mDO1=@{+tlz5$Wg4Y*LzwcRiE^Bl zG}fEGz}sOFE5^=l=KDH1t9==C&=1BC9q&v0nzo3TxbpFBr$Lb28M5$9&+??Nm^)i} zMhiu17cB-59Vg!S0-xo#qOPrsq6S=HDiULO@#|;zC~{xUYGPmO zKX|d#QjuTe!R^$WgjKc0Nj$_qQ-cnycy3w~{-nVY6j@{Lim6oop2R}DK|#t>X^yd2 zVKu>$J^qT(D2dx?A$1%?^lUZ=+JtnQ+Es-Y5EL}eeAcTG-U1RX4e-FP>1TIAX?TYH zAKTe4n|MO(NCQksQC@~c~(e_u06 z**}VL#qXE1p1L!sTBOv10ChA4D%(mCxBWQl;8ShX^ltgkU9CLh-W!U=42{;Xd2RDz z8k>L5%;bq_?Njrc*6fvOI_F?5|T!Nuujz9{r!l# zjRMyzpE()t?44+_(vBIzU+z&K%<^74J6D)IM)H3XMc~`7&jYKvko4BfIT@@e!)W4svy`H|| z%uIRvo4C&zcVuwdY1ZKZ=~MK`4H6nSr8K8Px$H)06J@0iWRtLz<0(_7<>nAxGQOD& zJ^w0!BLHIHY(lLgs=WTC>%R0j7%hQnM7VW3itlQqur4HT+Gdq0siAkZNB$Ea&t6L9 zASRlm{AMz%Z}T0HlE5zfHls|4tWz{Px8X2+Lc2;+*g~y^PoZ`WHBD#<2*P3aZlxS< zgqC>|rF~wWzd0|L+0YMd<+7|Kl;8kJ4@Ybx9W)Wn0v|${zMb(R4&%rX{7 z=Z5oPz!+C5ajCzwX{E8LBK;&V5p&DRS9>|3ztd^z>XECeU2vCn4HI0>ppvh-hnYT( z&i!P+G(rwFS144HW@rJZ1%Ithp-~e)=DZ3UiojJ(lhS*wsI0}zy-qA*)EnQ9k8`;- z!ZE>%%AAY0IYt&^l}o0YwPo8cPgis*C5SIp(7?syDpgCh)JQ2OFU5aV`DH*L{ilDh zKW6}@&2UE7T;cRNP+lSmVjrUt^^Tt`BM^0qiZ;NS#mHx;X4_r!rx^XkDNr@ zGWr^X5iFLqV&zKlpQZUkF)=iighTa;4Mk;R+jKc9n}@q%K~Z$mr_=A;RkTWFPcvm9>W{LH z40ZjfSb?&W-~Vkf;1j7{Khny<`y_OGp4NIPEgv9BqnyeBq~ zMPDP29wm36v<#{fFa6iZt=lM!Ur<0dQd4bfV>s1k1^=4TTqj^TnPt(>+Fxbv!8dXR zc-b6QcWcz*QZNBWo&0n3qwKrnQ!}dM$*m;P$l%ofz{Uo1T8RjhY-j&|#qZqx2f=xk z#hIGY2iNL5MD7$1n8(U!L)zCJd+K0*;!|PJnnC{W8#j^VP8%~;{*F;#Nm6s4g3TGK zK+0eU#Dm*W;Fu7_M9r9e2YM3s&%@D1W(yUPnNrfRgr9WUpN+5IsaQsg;FVs13C(@c z6dYTli&)^maTCzlFQjTB9Z+Q;R|Y0*N3mVD!tcWL$|JXeJ0paA<;U2 zh2r-pb1{h7y;Iz)NxJo4TVh!0hvafx@lQB?skN*djY%vvOFXP#{k`!>S>AW_`%4kg znNXy#SPHkQ!YUW!OHXZuhBak@d3PR(BF<~o<#1A7ows7&hhRlerb^;vtFbyP5sP@A zU(`V9D^l(lFW}T@N&2Uz&2KOCfSv^HH7l^*G>% zlTx`H$8-H>J&dw8K>8%3!M3qWz6!&k62P(G$y)KHEt~-1+;nnMZp5W|>3P9TyTc1; zU3j+XF}cx}ewdxLWKWg(lie%(A~R4#{v%^5p>fpk2ld{%F0L%kA|jkimvpqOA83o2 z*-)^u#zbSyVn}H^{isdP;6vLW!X#~m;~wtWrY-XE?#g8TOr8@zydb~UokV<-w0<)o zXEKsd?Fx&5(Oogs!^ki4r@gv%tJuD}b6j~;IL^=N_nd1W?Ij%xF}K>x@|5V~jE&!{ zE|Hnuk{0(~%oeobs+<{L)HlD6VZ?#c@ETdS{(k4;lF4Zu?+BwDu0PR$CvHro~xoW04bj(4WzdhV}tNqxW^DnX872YQPu zPoKraV;{HbyL3Ixra1tzn(`eL_{O+EyhqPX6|LQI_HI4@3pyFOgpM0Wy&=M1Wv3nT zU~x{x0hUvXs-mm*t#N&YuklqoGqAxfSsAd*FZ(@Y_JATNUs)KjzDKJbYlRf;CTV6s z0Q}^W`jdgwdUSl;l47X3D)$Fh|Ch(@oXGc+}@7_(EPIflm*y0cCAXbIutHI9F+g4t6_Mypp4L?29QtFOcrn|vhm^2D56@Y29qWvu z+`X#(!JAeBvvVnMASyQ9SsK)Lh>}EjWFfnNXy-WEhPTuz5r>lE%>=ECo+a_Qs}ndsrQp%O4cRaD`GK0A7I$Fv{#h`lq5q`6{$pdC2`5V;yK)11NVkfq>GR z@;Oo@a_Bvl@GjMGVmv;kRz|g1^5l$6OXXum{)7` zix!MsCKk{7C1KOFDw!N7BJ}pSP!((7#0wsf$SPOJR}l;mQxCceP+SW~HS8LBYf$U5 zOGw4VNvN3^)7E*3mJc(|A!+Uv@N5aDq)vvQ{IVCigW{qYmrd{#r%Ch(#W$6vU7YJ; zz}oTXyQJCqeF^5O{Thk_pl^BlR#+>@0h?7WSbrAC!~gO8{LrJfkdwXkfr*`0+TMd9 z(~eE!T)bDI=?ee%3X1f;)vKGf-onCr{kYiW*z_D0kh9y|5wF;b#&fh^)2NWfW+iQq z_vOX*K6udJFQZt##TDi4mt>t4kq9|@Hq9z^UG$4vKZa2+nM1JP^4t3HREro=yar85 zXhg?QoQXlql}LZNVN=#;v?^aQyf@V(`K%b{1waiiD;6-sc%dNcceZysn6$fHY>pa> zK+^OaY*r!KrEa}PwifoTcYp}(2myUGr5UDH=-jivs(-a{H6t^=%@BQD#!xigR8h!{ z;D#8jtU@PiupI^0o|bDLMW6>+3TixvS`w=@OsCc#POKuoYYoMh(BEWQttX+;Bh{Zp zIB4}=oXH<}8GQ691zDK_hqxNRm8U10Cx#>^Z6OaQ?pWE_Q(ttR?9bkhGeg|(x9iJW zjr*H!9go2eFBFWKiJ}5$CDnK9zupfUIOLMutnMhBLvGhQ^d9|OyJsT8%gk1wK0-i% zyDk*VvhnbP$6WC^a1jh-Ss9aNjnSUpi%BgWx+R+emybR$w8Yi7pt0nyAigV+O`~<* zDxe*v3a$tG{j@R<7FTDn9byr)qyfWg&1$mqap=Ul^BfC$$$1??0nxvR_p44^i>rzLn&)yOKyR#@yIIO&IVNzwaZQc!1;8eVzaZ<9Es?sl5&w<&N+ z4@fT+ogHZ(iadC8y>O{U%FeW4*lBN-^n-%qdFagtfAEP1GNa^P)E()8sS$3H`}^s- z9XDq=Lsd7L!3JrkV(wJl1YQtE$y88x;Rr}LA)`hS;CnyTBDu1SM8ifl!jYsrS)TaS z%7yHsPFs`aa8h_kzU*rpULtwklpDCa>@y~$c+}hT3%ADaN(wpSEGo=m1RvT6v$sJgCsA) z6C5(Cx>FqP)S%Z%x*tpqU`8VakkXvuswm=$={)*;lRu}URuq(BCRRDomBp~l3*r%V zY*4ANb=a|m9kmbJ+l?38AUzZOvU;0Hr(Uo{*0O6%GIAj2Zn-O+Kt(QDoYeW7n@qNN z8L-|#wzYdwmhj`yVt+*wQntGrV85DC&!}$%>a)!~j=Z;{pkhpLSY2p7+do;s-gB40 zYQ!GB5}1H`&`+TC{&b=j$B_`ljP^Zh#{z1Q-zLmzSK@4~9}d>#Xb6^?-d7*m&l$2x zm)Q|F)!>E;h|=tmYy8F_%bK;0-1X!erQyC4D(CDCG9I0Hunx#&73JIko-%!CP7nR4q8b!KYt_Bv56Ui2?iTFm|Z)5Zj!cwVcIFICR!R`5Lc>P)m*cv7f8vR7Z zrUx0DZpIVRYcO~|IzbO6` z7I3}rG={Pq>s}!oZWYzkAA?E3#bhl?M@Z&@6)~rj!@|v5)jefjDI?LLD4>M7bOu8j zab87dUo$P6(r@IOjG3>__0ALfG&f)|GE;P}*E|!$0)v_+kgqen+#2=#J<*_B$+M$VZ@ zDE%u@Kfgc735>bQBg5k&A$pi8anDngdbF*;0|Z=Dv97bI*v5?7b|k=PtvX!h-}pNS zR$716opUeI@@>u5Tzn&bg%Y$y60>eGEo^z+xbk{^{wh~2qA@%#r_fD19t2}mbw5iq zGkf|o9pd;&y*^c|N5EM;HG+%^nS=hlU@NTFGjC3>X2>-0cWWF~2+dJE|wb8GONs&J^j zs0nO48PqjBI`R>(v8Bykm0tL4!VKbABvPKtvFh4v*jaO+HS;LOFrh1huM84_HJgyy zR~a$G`2tKrnth_w-pabD>xmW(8xV~w5yVBg&vZ}fo^F|zf_mY?LhhLFZq3qk$=w=8 z9v-8FUVXaLCc4AO6)eLM;=)gcwY7#84Sjw9692MEt;8=yT6|aFDwCzKmy7@&8}-$9 z(yrN%1ujt|*thc%#AP;Gf=H?p?OP6lqlXEN6$N|w-$IzdKBjT#sa15S4&iD@q&{%f zmMvPBPV}zC88?+-i6!wJLA~nfhO!t`iU0-E3&las&A~bvvTqW$Lo_+%^B|VII28=h zXhP?v%l0L}H`SQ-%6AmDhkL`e3&a4RGL2={YGoA@!(3z+HW|@(4REKp^N=ZLRmpGeZ1z=V{oo_m`{ z1Juv+E<>lY>ME@LBgA!BjZO0KQX%VzWf~k2RNnK z{m8VObg2VgS(_|hF7iYSF0_FQ#_!#;Uc6eC+RtNR&HyZSfPzvp--u)QXIAb8i2Wf= znF#6rF(&0%-^?SGlqV?bPkby{^a^Bxit|vjQlra5Hgv$N_Ew#$jURx$FV`i1+OFA| zn<+RYmx*rQdPY4fICVGi*gTu5rHA$!+qXUDRIT|U3YOLS^yvSm(g?!sjl$c{Z>PH8 zZ7le;kkh-9v8(UyFNQ%uRHsbotzA(PY2z!nnrL^Yv zq{JFw;m!H8Z#~t5iXL>fK^EZ6hsVZX1s3t8dPraW&PTCCWiArg=rrOf6KsD^JcN00 z(XM^ag*F|&D!GeY{KY29XAc+4TsIsWI4=bB!u9!{ zaMuuzemX`ZnsiRUknh{xk{%1PSp5aZ8NX8=ze6@=azksa^-hH?#rMknI>{5~dM6>@ zi=HmlwMR4+3D(hUYvEN6m)_*L%>2KZ*@Wv0trw@4$c3OGlnRt3_peFdNS z$e3|6u(J~mx1&AzP!Q`1ru^~?`~b>*^?v6NzaU?%RldVip6UrF(!9#l0ZhXw$4F zTSFVH!6R!>{C6eXy(Yc2Y?FO=k59aUvpG>sq1_7x>Z^owponEJlB1^|yUb*oJTxrs{`w@V=6fVE^Kwkj`4WF+;!o0Bk4$ zRfKaFYIu%Ab;iDeP^UdBti55%R38l2J3DH@z5aHxM=qdt_Sf~DJ5^1(rJn|l@;!JU z)|3hnMYu^FHx$0c>!xyb(vi)Caso33jp|6lbL6hW7=%v;w^q4>UBA-xjBRf$*kUD6S12{JoF zJO^ta@yDRGC|J?Cu>u*zX=lE7z^7%?k%p!W67D9<{Dp*gV-#_h=d6c%3JSMT9~CE>=cfH5Nl<6=%rpmGMt& zJG}W$o}xlQoQ^3cJ8VQ{yy-Q*^J>Jy11I%{t=s4UOT>RB!ZSK!I{D(QOeV&QmVhFB z=lcT8*5tF*zlh~zAgd@z1J9=_&DIdEth_2L<&ofuf2f)&sA!HsPKeRe2ng${2nd=g qh)4to|JV2b-`fB9{{Juj|Gxk{eg#QCFtq1-9 literal 0 HcmV?d00001 diff --git a/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/test_ercot_daily_load_iso.py b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/test_ercot_daily_load_iso.py new file mode 100644 index 000000000..05c1bb0fe --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/pipelines/sources/spark/iso/test_ercot_daily_load_iso.py @@ -0,0 +1,1233 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys + +sys.path.insert(0, ".") +import pandas as pd +from requests import HTTPError + +import pytest +from pyspark.sql import DataFrame, SparkSession +from pytest_mock import MockerFixture + +from src.sdk.python.rtdip_sdk.pipelines.sources.spark.iso import ERCOTDailyLoadISOSource +from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.iso import ERCOT_SCHEMA +from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import Libraries + +dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") + +iso_configuration = { + "load_type": "actual", + "date": "2023-11-17", + "certificate_pfx_key": "SOME_KEY", + "certificate_pfx_key_contents": "SOME_DATA", +} + +patch_module_name = "requests.get" + +url_actual = "https://mis.ercot.com/misapp/GetReports.do?reportTypeId=13101" + + +# Had to put HTML Content in the Python code as Sonar was raising Bugs/Code-Smells while putting it in an HTML file. + +urls_content_actual = """ERCOT MIS
Actual System Load by Weather Zone
Title
XML
CSV
Other
cdr.00013101.0000000000000000.20231120.055000675.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231120.055000624.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231119.055001313.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231119.055001249.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231118.055000923.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231118.055000878.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231117.055001328.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231117.055001289.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231116.055001166.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231116.055001116.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231115.055000366.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231115.055000320.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231114.055001063.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231114.055001023.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231113.055000961.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231113.055000899.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231112.055000824.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231112.055000755.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231111.055001050.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231111.055000997.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231110.055000881.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231110.055000807.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231109.055001496.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231109.055001428.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231108.055000833.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231108.055000763.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231107.055001060.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231107.055000955.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231106.055001015.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231106.055000774.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231105.055000406.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231105.055000352.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231104.055000981.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231104.055000902.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231103.055000827.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231103.055000785.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231102.055000881.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231102.055000837.ACTUALSYSLOADWZNP6345_xml.zip
cdr.00013101.0000000000000000.20231101.055000731.ACTUALSYSLOADWZNP6345_csv.zip
cdr.00013101.0000000000000000.20231101.055000678.ACTUALSYSLOADWZNP6345_xml.zip
October + 2023
""" + + +urls_content_forecast = """ERCOT MIS
Seven-Day Load Forecast by Weather Zone
Title
XML
CSV
Other
cdr.00012312.0000000000000000.20231121.113000947.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.113000870.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.103000844.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.103000795.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.093000715.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.093000620.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.083000978.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.083000647.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.073000714.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.073000638.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.063000996.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.063000933.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.053000991.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.053000916.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.043000722.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.043000660.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.033000594.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.033000524.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.023000647.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.023000579.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.013000671.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.013000605.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231121.003002949.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231121.003002883.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.233000886.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.233000832.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.223000746.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.223000674.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.213000634.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.213000555.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.203003862.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.203003749.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.193003016.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.193002932.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.183000767.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.183000704.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.173000910.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.173000818.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.163000873.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.163000803.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.153000703.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.153000639.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.143000805.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.143000717.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.133000876.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.133000799.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.123000704.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.123000636.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.113000937.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.113000860.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.103000931.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.103000829.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.093000786.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.093000711.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.083000958.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.083000887.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.073000657.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.073000564.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.063000700.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.063000612.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.053000867.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.053000763.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.043001060.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.043000971.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.033000790.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.033000733.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.023003151.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.023003021.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.013004958.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.013004846.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231120.003000884.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231120.003000805.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.233000754.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.233000680.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.223000763.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.223000687.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.213000937.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.213000883.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.203002843.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.203002748.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.193001076.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.193000972.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.183004389.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.183004280.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.173000833.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.173000741.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.163000807.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.163000693.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.153002884.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.153002766.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.143001054.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.143000982.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.133000741.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.133000677.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.123001145.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.123000986.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.113000994.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.113000923.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.103000854.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.103000794.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.093000799.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.093000713.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.083003124.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.083003023.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.073000801.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.073000684.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.063000823.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.063000746.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.053000703.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.053000625.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.043000822.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.043000746.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.033002584.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.033002524.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.023001118.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.023001048.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.013002886.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.013002823.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231119.003001271.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231119.003001202.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.233001692.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.233001611.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.223003232.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.223003115.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.213003531.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.213003251.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.203001419.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.203001328.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.193001023.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.193000949.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.183000805.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.183000735.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.173000807.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.173000712.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.163000824.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.163000757.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.153001492.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.153001406.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.143000792.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.143000724.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.133002938.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.133002874.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.123001841.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.123001776.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.113001554.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.113001463.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.103000596.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.103000528.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.093003526.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.093003472.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.083000810.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.083000756.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.073001590.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.073001464.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.063000926.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.063000849.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.053001065.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.053000992.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.043000916.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.043000836.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.033003455.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.033003403.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.023000841.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.023000773.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.013003392.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.013003286.LFCWEATHERNP3561_xml.zip
cdr.00012312.0000000000000000.20231118.003000791.LFCWEATHERNP3561_csv.zip
cdr.00012312.0000000000000000.20231118.003000726.LFCWEATHERNP3561_xml.zip
""" + + +def test_ercot_daily_load_iso_read_setup(spark_session: SparkSession): + iso_source = ERCOTDailyLoadISOSource(spark_session, iso_configuration) + + assert iso_source.system_type().value == 2 + assert iso_source.libraries() == Libraries( + maven_libraries=[], pypi_libraries=[], pythonwheel_libraries=[] + ) + + assert isinstance(iso_source.settings(), dict) + expected_values = [ + "certificate_pfx_key", + "certificate_pfx_key_contents", + "date", + "load_type", + ] + assert sorted(iso_source.required_options) == expected_values + assert iso_source.pre_read_validation() + + +def mock_certificates(mocker): + class Key: + @staticmethod + def private_bytes(*args, **kwargs) -> bytes: + return b"" + + class Cert: + @staticmethod + def public_bytes(*args, **kwargs) -> bytes: + return b"" + + mocker.patch( + "cryptography.hazmat.primitives.serialization.pkcs12.load_key_and_certificates", + side_effect=lambda *args, **kwargs: (Key(), Cert(), ""), + ) + + +def ercot_daily_load_iso_read_batch_test( + spark_session: SparkSession, + mocker: MockerFixture, + load_type: str, + url_to_match: str, + file_download_url_to_match: str, + urls_bytes: bytes, + zip_file: str, + date: str, + expected_data_file: str, + expected_rows_count: int = 24, +): + iso_source = ERCOTDailyLoadISOSource( + spark_session, + {**iso_configuration, "load_type": load_type, "date": date}, + ) + + with open(f"{dir_path}/{zip_file}", "rb") as file: + zip_download_bytes = file.read() + + class URLsResponse: + content = urls_bytes + status_code = 200 + + class ZipDownloadResponse(URLsResponse): + content = zip_download_bytes + + def get_response(url: str, *args, **kwargs): + if url == url_to_match: + return URLsResponse() + else: + assert url == file_download_url_to_match + return ZipDownloadResponse() + + mocker.patch(patch_module_name, side_effect=get_response) + mock_certificates(mocker) + + df = iso_source.read_batch() + + assert df.count() == expected_rows_count + assert isinstance(df, DataFrame) + assert str(df.schema) == str(ERCOT_SCHEMA) + + expected_df_spark = spark_session.createDataFrame( + pd.read_csv(f"{dir_path}/{expected_data_file}", parse_dates=["Date"]), + schema=ERCOT_SCHEMA, + ) + + cols = df.columns + assert df.orderBy(cols).collect() == expected_df_spark.orderBy(cols).collect() + + +def test_ercot_daily_load_iso_read_batch_actual( + spark_session: SparkSession, mocker: MockerFixture +): + ercot_daily_load_iso_read_batch_test( + spark_session, + mocker, + load_type="actual", + url_to_match=url_actual, + file_download_url_to_match=( + "https://mis.ercot.com/misdownload/servlets/mirDownload?mimic_duns=1118490502000" + "&doclookupId=959096531" + ), + urls_bytes=urls_content_actual.encode("utf-8"), + zip_file="ercot_daily_load_actual_sample1.zip", + expected_data_file="ercot_daily_load_actual_expected.csv", + date="2023-11-17", + ) + + +def test_ercot_daily_load_iso_read_batch_forecast( + spark_session: SparkSession, mocker: MockerFixture +): + ercot_daily_load_iso_read_batch_test( + spark_session, + mocker, + load_type="forecast", + url_to_match="https://mis.ercot.com/misapp/GetReports.do?reportTypeId=12312", + file_download_url_to_match=( + "https://mis.ercot.com/misdownload/servlets/mirDownload?mimic_duns=1118490502000" + "&doclookupId=959500285" + ), + urls_bytes=urls_content_forecast.encode("utf-8"), + zip_file="ercot_daily_load_forecast_sample1.zip", + date="2023-11-19", + expected_data_file="ercot_daily_load_forecast_expected.csv", + expected_rows_count=192, + ) + + +def test_ercot_daily_load_iso_empty_response( + spark_session: SparkSession, mocker: MockerFixture +): + iso_source = ERCOTDailyLoadISOSource( + spark_session, + {**iso_configuration, "load_type": "actual", "date": "2023-11-16"}, + ) + + class EmptyURLsResponse: + content = urls_content_actual + status_code = 200 + + class EmptyZipDownloadResponse(EmptyURLsResponse): + content = b"" + + def get_response(url: str, *args, **kwargs): + if url == url_actual: + return EmptyURLsResponse() + else: + return EmptyZipDownloadResponse() + + mocker.patch(patch_module_name, side_effect=get_response) + mock_certificates(mocker) + + with pytest.raises(HTTPError) as exc_info: + iso_source.read_batch() + + assert str(exc_info.value) == "Empty Response was returned" + + +def test_ercot_daily_load_iso_no_file( + spark_session: SparkSession, mocker: MockerFixture +): + iso_source = ERCOTDailyLoadISOSource( + spark_session, + {**iso_configuration, "date": "2023-09-16"}, + ) + + class NoFileURLsResponse: + content = urls_content_actual + status_code = 200 + + def get_response(url: str, *args, **kwargs): + if url == url_actual: + return NoFileURLsResponse() + + mocker.patch(patch_module_name, side_effect=get_response) + mock_certificates(mocker) + + with pytest.raises(ValueError) as exc_info: + iso_source.read_batch() + + assert str(exc_info.value) == "No file was found for date - 20230917" + + +def test_ercot_daily_load_iso_no_data( + spark_session: SparkSession, mocker: MockerFixture +): + iso_source = ERCOTDailyLoadISOSource( + spark_session, + {**iso_configuration}, + ) + + with open(f"{dir_path}/ercot_daily_load_actual_sample2.zip", "rb") as file: + no_data_zip_download_bytes = file.read() + + class NoDataURLsResponse: + content = urls_content_actual + status_code = 200 + + class NoDataZipDownloadResponse(NoDataURLsResponse): + content = no_data_zip_download_bytes + + def get_response(url: str, *args, **kwargs): + if url == url_actual: + return NoDataURLsResponse() + else: + return NoDataZipDownloadResponse() + + mocker.patch(patch_module_name, side_effect=get_response) + mock_certificates(mocker) + + with pytest.raises(ValueError) as exc_info: + iso_source.read_batch() + + assert str(exc_info.value) == "No data was found in the specified interval" + + +def test_ercot_daily_load_iso_iso_invalid_date(spark_session: SparkSession): + with pytest.raises(ValueError) as exc_info: + iso_source = ERCOTDailyLoadISOSource( + spark_session, {**iso_configuration, "date": "2023/11/01"} + ) + iso_source.pre_read_validation() + + expected = "Unable to parse date. Please specify in %Y-%m-%d format." + assert str(exc_info.value) == expected