diff --git a/uk_bin_collection/tests/council_schemas/EastSuffolkCouncil.schema b/uk_bin_collection/tests/council_schemas/EastSuffolkCouncil.schema new file mode 100644 index 0000000000..f77b5ec089 --- /dev/null +++ b/uk_bin_collection/tests/council_schemas/EastSuffolkCouncil.schema @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/Welcome5", + "definitions": { + "Welcome5": { + "type": "object", + "additionalProperties": false, + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/definitions/Bin" + } + } + }, + "required": [ + "bins" + ], + "title": "Welcome5" + }, + "Bin": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "collectionDate": { + "type": "string" + } + }, + "required": [ + "collectionDate", + "type" + ], + "title": "Bin" + } + } +} diff --git a/uk_bin_collection/tests/features/validate_council_outputs.feature b/uk_bin_collection/tests/features/validate_council_outputs.feature index c9a1da1d5b..9a796630a0 100644 --- a/uk_bin_collection/tests/features/validate_council_outputs.feature +++ b/uk_bin_collection/tests/features/validate_council_outputs.feature @@ -34,6 +34,7 @@ Feature: Test each council output matches expected results in /outputs | EastleighBoroughCouncil | | EastNorthamptonshireCouncil | | EastRidingCouncil | + | EastSuffolkCouncil | | ErewashBoroughCouncil | | FenlandDistrictCouncil | | GlasgowCityCouncil | diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index f9d3713841..7baf6be3ee 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -166,6 +166,14 @@ "url": "https://wasterecyclingapi.eastriding.gov.uk", "wiki_name": "East Riding Council" }, + "EastSuffolkCouncil": { + "SKIP_GET_URL": "SKIP_GET_URL", + "uprn": "10093544720", + "postcode": "IP11 9FJ" + "url": "https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder", + "wiki_name": "East Suffolk Council", + "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" + }, "ErewashBoroughCouncil": { "SKIP_GET_URL": "SKIP_GET_URL", "uprn": "10003582028", @@ -582,4 +590,4 @@ "url": "https://waste-api.york.gov.uk/api/Collections/GetBinCollectionDataForUprn/", "wiki_name": "York Council" } -} \ No newline at end of file +} diff --git a/uk_bin_collection/tests/outputs/EastSuffolkCouncil.json b/uk_bin_collection/tests/outputs/EastSuffolkCouncil.json new file mode 100644 index 0000000000..52c2433620 --- /dev/null +++ b/uk_bin_collection/tests/outputs/EastSuffolkCouncil.json @@ -0,0 +1,12 @@ +{ + "bins": [ + { + "type": "General waste", + "collectionDate": "20/10/2023" + }, + { + "type": "Recycling", + "collectionDate": "27/10/2023" + } + ] +} diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py new file mode 100644 index 0000000000..c3497ab7e7 --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -0,0 +1,99 @@ +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait, Select +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import \ + AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the base + class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + # Set up Selenium to run 'headless' + options = webdriver.ChromeOptions() + options.add_argument("--headless") + options.add_argument("--no-sandbox") + options.add_argument("--disable-gpu") + options.add_argument("--disable-dev-shm-usage") + options.add_experimental_option("excludeSwitches", ["enable-logging"]) + + user_uprn = kwargs.get("uprn") + user_postcode = kwargs.get("postcode") + check_uprn(user_uprn) + check_postcode(user_postcode) + + # Create Selenium webdriver + driver = webdriver.Chrome(options=options) + driver.get("https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder") + + # Wait for iframe to load and switch to it + WebDriverWait(driver, 30).until(EC.frame_to_be_available_and_switch_to_it((By.ID, 'fillform-frame-1'))) + + # Wait for postcode entry box + postcode = WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.ID, "alt_postcode_search")) + ) + # Enter postcode + postcode.send_keys(user_postcode) + + # Wait for address selection dropdown to appear + address = Select( + WebDriverWait(driver, 10).until( + EC.visibility_of_element_located((By.ID, "alt_choose_address")) + ) + ) + + # Wait for spinner to disappear (signifies options are loaded for select) + WebDriverWait(driver, 10).until( + EC.invisibility_of_element_located((By.CLASS_NAME, "spinner-outer")) + ) + + # Select address by UPRN + address.select_by_value(user_uprn) + + # Wait for spinner to disappear (signifies data is loaded) + WebDriverWait(driver, 10).until( + EC.invisibility_of_element_located((By.CLASS_NAME, "spinner-outer")) + ) + + # Find data table + data_table = WebDriverWait(driver, 10).until( + EC.presence_of_element_located( + ( + By.XPATH, + '//div[@data-field-name="collection_details"]/div[contains(@class, "fieldContent")]/div[contains(@class, "repeatable-table-wrapper")]', + ) + ) + ) + + + + # Make a BS4 object + soup = BeautifulSoup(data_table.get_attribute("innerHTML"), features="html.parser") + + data = {"bins": []} + + rows = soup.find("table").find("tbody").find_all("tr") + for row in rows: + cols = row.find_all("td") + bin_type = cols[2].find_all("span")[1].text + collection_date = cols[3].find_all("span")[1].text + collection_date = datetime.strptime(collection_date, "%d/%m/%Y").strftime( + date_format + ) + dict_data = {"type": bin_type, "collectionDate": collection_date} + data["bins"].append(dict_data) + + data["bins"].sort( + key=lambda x: datetime.strptime(x.get("collectionDate"), date_format) + ) + + return data