From 764817944fb4a0816f898709ad9c9ff5af4a9820 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Mon, 21 Oct 2024 23:19:12 +0100 Subject: [PATCH 1/5] fix: Southwark Council --- uk_bin_collection/tests/input.json | 4 ++-- .../uk_bin_collection/councils/SouthwarkCouncil.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index aea6f8dc8a..33c5981712 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1141,8 +1141,8 @@ "wiki_name": "South Tyneside Council" }, "SouthwarkCouncil": { - "url": "https://www.southwark.gov.uk/bins/lookup/", - "wiki_command_url_override": "https://www.southwark.gov.uk/bins/lookup/XXXXXXXX", + "url": "https://services.southwark.gov.uk/bins/lookup/", + "wiki_command_url_override": "https://services.southwark.gov.uk/bins/lookup/XXXXXXXX", "uprn": "200003469271", "wiki_name": "Southwark Council", "wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." diff --git a/uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py index 46361eefe3..cef578cf69 100644 --- a/uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py @@ -19,7 +19,7 @@ def parse_data(self, page: str, **kwargs) -> dict: check_uprn(user_uprn) data = {"bins": []} - baseurl = "https://www.southwark.gov.uk/bins/lookup/" + baseurl = "https://services.southwark.gov.uk/bins/lookup/" url = baseurl + user_uprn headers = { @@ -74,9 +74,13 @@ def parse_data(self, page: str, **kwargs) -> dict: data["bins"].append(dict_data) # Extract food waste collection information - food_section = soup.find("div", {"aria-labelledby": "organicsCollectionTitle"}) + food_section = soup.find( + "div", {"aria-labelledby": "domesticFoodCollectionTitle"} + ) if food_section: - food_title = food_section.find("p", {"id": "organicsCollectionTitle"}).text + food_title = food_section.find( + "p", {"id": "domesticFoodCollectionTitle"} + ).text food_next_collection = ( food_section.find(text=lambda text: "Next collection" in text) .strip() From 1d5221a22e972ddecb0793d8aedb8a616ad17b14 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Mon, 21 Oct 2024 23:52:08 +0100 Subject: [PATCH 2/5] fix: West Berkshire Council --- .../councils/WestBerkshireCouncil.py | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py index a6d8914034..8693a13592 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestBerkshireCouncil.py @@ -77,44 +77,34 @@ def parse_data(self, page: str, **kwargs) -> dict: rubbish_div = soup.find( "div", {"id": "FINDYOURBINDAYS_RUBBISHDATE_OUTERDIV"} ) - try: - rubbish_date = rubbish_div.find_all("div")[2] - rubbish_date = datetime.strptime( - rubbish_date.text, - "%A %d %B", - ).replace(year=datetime.now().year) - except: - rubbish_date = rubbish_div.find_all("div")[3] + rubbish_date = rubbish_div.find_all("div")[2] + if rubbish_date.text == "Today": + rubbish_date = datetime.now() + else: rubbish_date = datetime.strptime( rubbish_date.text, "%A %d %B", ).replace(year=datetime.now().year) + recycling_div = soup.find( "div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"} ) - try: - recycling_date = recycling_div.find_all("div")[2] + recycling_date = recycling_div.find_all("div")[2] + if recycling_date.text == "Today": + recycling_date = datetime.now() + else: recycling_date = datetime.strptime( recycling_date.text, "%A %d %B", ).replace(year=datetime.now().year) - except: - rubbish_date = recycling_div.find_all("div")[3] - rubbish_date = datetime.strptime( - rubbish_date.text, - "%A %d %B", - ).replace(year=datetime.now().year) + food_div = soup.find( "div", {"id": "FINDYOURBINDAYS_RECYCLINGDATE_OUTERDIV"} ) - try: - food_date = food_div.find_all("div")[2] - food_date = datetime.strptime( - food_date.text, - "%A %d %B", - ).replace(year=datetime.now().year) - except: - food_date = food_div.find_all("div")[3] + food_date = food_div.find_all("div")[2] + if food_date.text == "Today": + food_date = datetime.now() + else: food_date = datetime.strptime( food_date.text, "%A %d %B", From 2985f666c9b6fd3e555ec9ca798cb1640546c5f2 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:46:24 +0000 Subject: [PATCH 3/5] feat: Adding Flintshire County Council fix: #750 --- uk_bin_collection/tests/input.json | 7 +++ .../councils/FlintshireCountyCouncil.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 uk_bin_collection/uk_bin_collection/councils/FlintshireCountyCouncil.py diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 64e5286106..4fa68896dc 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -484,6 +484,13 @@ "url": "https://www.fenland.gov.uk/article/13114/", "wiki_name": "Fenland District Council" }, + "FlintshireCountyCouncil": { + "url": "https://digital.flintshire.gov.uk", + "wiki_command_url_override": "https://digital.flintshire.gov.uk", + "uprn": "100100213710", + "wiki_name": "Flintshire County Council", + "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." + }, "ForestOfDeanDistrictCouncil": { "house_number": "ELMOGAL, PARKEND ROAD, BREAM, LYDNEY", "postcode": "GL15 6JT", diff --git a/uk_bin_collection/uk_bin_collection/councils/FlintshireCountyCouncil.py b/uk_bin_collection/uk_bin_collection/councils/FlintshireCountyCouncil.py new file mode 100644 index 0000000000..486ea67739 --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/FlintshireCountyCouncil.py @@ -0,0 +1,60 @@ +import requests +from bs4 import BeautifulSoup + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + + user_uprn = kwargs.get("uprn") + check_uprn(user_uprn) + bindata = {"bins": []} + + URI = f"https://digital.flintshire.gov.uk/FCC_BinDay/Home/Details2/{user_uprn}" + + # Make the GET request + response = requests.get(URI) + + # Parse the HTML content + soup = BeautifulSoup(response.content, "html.parser") + + # Adjust these tags and classes based on actual structure + # Example for finding collection dates and types + bin_collections = soup.find_all( + "div", class_="col-md-12 col-lg-12 col-sm-12 col-xs-12" + ) # Replace with actual class name + + # Extracting and printing the schedule data + schedule = [] + for collection in bin_collections: + dates = collection.find_all("div", class_="col-lg-2 col-md-2 col-sm-2") + bin_type = collection.find("div", class_="col-lg-3 col-md-3 col-sm-3") + + if dates[0].text.strip() == "Date of Collection": + continue + + bin_types = bin_type.text.strip().split(" / ") + date = dates[0].text.strip() + + # Loop through the dates for each collection type + for bin_type in bin_types: + + dict_data = { + "type": bin_type, + "collectionDate": date, + } + bindata["bins"].append(dict_data) + + bindata["bins"].sort( + key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y") + ) + return bindata From 297f7e364003daf9bcdbf6a645a26619955bb938 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:08:37 +0000 Subject: [PATCH 4/5] feat: Adding Fife Council fix: #933 --- uk_bin_collection/tests/input.json | 7 ++ .../uk_bin_collection/councils/FifeCouncil.py | 68 +++++++++++++++++++ wiki/Councils.md | 37 ++++++++-- 3 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 uk_bin_collection/uk_bin_collection/councils/FifeCouncil.py diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 4fa68896dc..2507f31f55 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -484,6 +484,13 @@ "url": "https://www.fenland.gov.uk/article/13114/", "wiki_name": "Fenland District Council" }, + "FifeCouncil": { + "url": "https://www.fife.gov.uk", + "wiki_command_url_override": "https://www.fife.gov.uk", + "uprn": "320203521", + "wiki_name": "Fife Council", + "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." + }, "FlintshireCountyCouncil": { "url": "https://digital.flintshire.gov.uk", "wiki_command_url_override": "https://digital.flintshire.gov.uk", diff --git a/uk_bin_collection/uk_bin_collection/councils/FifeCouncil.py b/uk_bin_collection/uk_bin_collection/councils/FifeCouncil.py new file mode 100644 index 0000000000..cbf04f347a --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/FifeCouncil.py @@ -0,0 +1,68 @@ +from datetime import datetime + +import requests +from bs4 import BeautifulSoup + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + # Get and check UPRN + user_uprn = kwargs.get("uprn") + check_uprn(user_uprn) + bindata = {"bins": []} + + API_URL = "https://www.fife.gov.uk/api/custom?action=powersuite_bin_calendar_collections&actionedby=bin_calendar&loadform=true&access=citizen&locale=en" + AUTH_URL = "https://www.fife.gov.uk/api/citizen?preview=false&locale=en" + AUTH_KEY = "Authorization" + + r = requests.get(AUTH_URL) + r.raise_for_status() + auth_token = r.headers[AUTH_KEY] + + post_data = { + "name": "bin_calendar", + "data": { + "uprn": user_uprn, + }, + "email": "", + "caseid": "", + "xref": "", + "xref1": "", + "xref2": "", + } + + headers = { + "referer": "https://www.fife.gov.uk/services/forms/bin-calendar", + "accept": "application/json", + "content-type": "application/json", + AUTH_KEY: auth_token, + } + + r = requests.post(API_URL, data=json.dumps(post_data), headers=headers) + r.raise_for_status() + + result = r.json() + + for collection in result["data"]["tab_collections"]: + dict_data = { + "type": collection["colour"], + "collectionDate": datetime.strptime( + collection["date"], + "%A, %B %d, %Y", + ).strftime("%d/%m/%Y"), + } + bindata["bins"].append(dict_data) + + bindata["bins"].sort( + key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y") + ) + return bindata diff --git a/wiki/Councils.md b/wiki/Councils.md index 8244d8e058..e78eb2732d 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -77,6 +77,8 @@ This document is still a work in progress, don't worry if your council isn't lis - [Falkirk Council](#falkirk-council) - [Fareham Borough Council](#fareham-borough-council) - [Fenland District Council](#fenland-district-council) +- [Fife Council](#fife-council) +- [Flintshire County Council](#flintshire-county-council) - [Forest of Dean District Council](#forest-of-dean-district-council) - [Gateshead Council](#gateshead-council) - [Gedling Borough Council](#gedling-borough-council) @@ -996,6 +998,28 @@ Additional parameters: --- +### Fife Council +```commandline +python collect_data.py FifeCouncil https://www.fife.gov.uk -u XXXXXXXX +``` +Additional parameters: +- `-u` - UPRN + +Note: You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. + +--- + +### Flintshire County Council +```commandline +python collect_data.py FlintshireCountyCouncil https://digital.flintshire.gov.uk -u XXXXXXXX +``` +Additional parameters: +- `-u` - UPRN + +Note: You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. + +--- + ### Forest of Dean District Council ```commandline python collect_data.py ForestOfDeanDistrictCouncil https://community.fdean.gov.uk/s/waste-collection-enquiry -s -p "XXXX XXX" -n XX -w http://HOST:PORT/ @@ -1421,10 +1445,14 @@ Note: Pass the house name/number plus the name of the street with the postcode p ### Midlothian Council ```commandline -python collect_data.py MidlothianCouncil https://www.midlothian.gov.uk/directory_record/XXXXXX/XXXXXX +python collect_data.py MidlothianCouncil https://www.midlothian.gov.uk/info/1054/bins_and_recycling/343/bin_collection_days -s -p "XXXX XXX" -n XX ``` +Additional parameters: +- `-s` - skip get URL +- `-p` - postcode +- `-n` - house number -Note: Follow the instructions [here](https://www.midlothian.gov.uk/info/1054/bins_and_recycling/343/bin_collection_days) until you get the page that shows the weekly collections for your address then copy the URL and replace the URL in the command. +Note: Pass the house name/number wrapped in double quotes along with the postcode parameter --- @@ -2027,7 +2055,7 @@ Additional parameters: ### Southwark Council ```commandline -python collect_data.py SouthwarkCouncil https://www.southwark.gov.uk/bins/lookup/XXXXXXXX -u XXXXXXXX +python collect_data.py SouthwarkCouncil https://services.southwark.gov.uk/bins/lookup/XXXXXXXX -u XXXXXXXX ``` Additional parameters: - `-u` - UPRN @@ -2191,10 +2219,11 @@ Additional parameters: ### Teignbridge Council ```commandline -python collect_data.py TeignbridgeCouncil https://www.google.co.uk -u XXXXXXXX +python collect_data.py TeignbridgeCouncil https://www.google.co.uk -u XXXXXXXX -w http://HOST:PORT/ ``` Additional parameters: - `-u` - UPRN +- `-w` - remote Selenium web driver URL (required for Home Assistant) Note: Provide Google as the URL as the real URL breaks the integration. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. From 82b7cd13b0ed189c2f20c18852b29baa0966f8d3 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Mon, 28 Oct 2024 00:30:52 +0000 Subject: [PATCH 5/5] feat: Adding Stockton On Tees Council fix: #930 --- uk_bin_collection/tests/input.json | 8 + .../councils/StocktonOnTeesCouncil.py | 159 ++++++++++++++++++ wiki/Councils.md | 13 ++ 3 files changed, 180 insertions(+) create mode 100644 uk_bin_collection/uk_bin_collection/councils/StocktonOnTeesCouncil.py diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 2507f31f55..bb898e4e83 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1196,6 +1196,14 @@ "wiki_name": "Stockport Borough Council", "wiki_note": "Replace XXXXXXXX with UPRN." }, + "StocktonOnTeesCouncil": { + "house_number": "24", + "postcode": "TS20 2RD", + "skip_get_url": true, + "url": "https://www.stockton.gov.uk", + "web_driver": "http://selenium:4444", + "wiki_name": "Stockton On Tees Council" + }, "StokeOnTrentCityCouncil": { "url": "https://www.stoke.gov.uk/jadu/custom/webserviceLookUps/BarTecWebServices_missed_bin_calendar.php?UPRN=3455121482", "wiki_command_url_override": "https://www.stoke.gov.uk/jadu/custom/webserviceLookUps/BarTecWebServices_missed_bin_calendar.php?UPRN=XXXXXXXXXX", diff --git a/uk_bin_collection/uk_bin_collection/councils/StocktonOnTeesCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StocktonOnTeesCouncil.py new file mode 100644 index 0000000000..2d2986f5fa --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/StocktonOnTeesCouncil.py @@ -0,0 +1,159 @@ +import time + +from bs4 import BeautifulSoup +from dateutil.relativedelta import relativedelta +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.wait import WebDriverWait + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + driver = None + try: + data = {"bins": []} + collections = [] + user_paon = kwargs.get("paon") + user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") + headless = kwargs.get("headless") + check_paon(user_paon) + check_postcode(user_postcode) + + # Create Selenium webdriver + driver = create_webdriver(web_driver, headless, None, __name__) + driver.get("https://www.stockton.gov.uk/bin-collection-days") + + # Wait for the postcode field to appear then populate it + inputElement_postcode = WebDriverWait(driver, 30).until( + EC.presence_of_element_located( + ( + By.ID, + "LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_ADDRESSLOOKUPPOSTCODE", + ) + ) + ) + inputElement_postcode.send_keys(user_postcode) + + # Click search button + findAddress = WebDriverWait(driver, 10).until( + EC.presence_of_element_located( + ( + By.ID, + "LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_ADDRESSLOOKUPSEARCH", + ) + ) + ) + findAddress.click() + + WebDriverWait(driver, 10).until( + EC.element_to_be_clickable( + ( + By.XPATH, + "" + "//*[@id='LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_ADDRESSLOOKUPADDRESS']//option[contains(., '" + + user_paon + + "')]", + ) + ) + ).click() + + # Wait for the submit button to appear, then click it to get the collection dates + WebDriverWait(driver, 30).until( + EC.presence_of_element_located( + ( + By.XPATH, + '//*[@id="LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_COLLECTIONDETAILS2"]/div', + ) + ) + ) + time.sleep(2) + + soup = BeautifulSoup(driver.page_source, features="html.parser") + soup.prettify() + + rubbish_div = soup.find( + "p", + { + "class": "myaccount-block__date myaccount-block__date--bin myaccount-block__date--waste" + }, + ) + rubbish_date = rubbish_div.text + if rubbish_date == "Today": + rubbish_date = datetime.now() + else: + rubbish_date = datetime.strptime( + remove_ordinal_indicator_from_date_string(rubbish_date).strip(), + "%a %d %B %Y", + ).replace(year=datetime.now().year) + + recycling_div = soup.find( + "p", + { + "class": "myaccount-block__date myaccount-block__date--bin myaccount-block__date--recycling" + }, + ) + recycling_date = recycling_div.text + if recycling_date == "Today": + recycling_date = datetime.now() + else: + recycling_date = datetime.strptime( + remove_ordinal_indicator_from_date_string(recycling_date).strip(), + "%a %d %B %Y", + ) + + garden_div = soup.find( + "div", + { + "class": "myaccount-block__item myaccount-block__item--bin myaccount-block__item--garden" + }, + ) + garden_date = garden_div.find("strong") + if garden_date.text.strip() == "Date not available": + print("Garden waste unavailable") + else: + if garden_date.text == "Today": + garden_date = datetime.now() + collections.append(("Garden waste bin", garden_date)) + else: + garden_date = datetime.strptime( + remove_ordinal_indicator_from_date_string( + garden_date.text + ).strip(), + "%a %d %B %Y", + ) + collections.append(("Garden waste bin", garden_date)) + + collections.append(("Rubbish bin", rubbish_date)) + collections.append(("Recycling bin", recycling_date)) + + ordered_data = sorted(collections, key=lambda x: x[1]) + for item in ordered_data: + dict_data = { + "type": item[0].capitalize(), + "collectionDate": item[1].strftime(date_format), + } + data["bins"].append(dict_data) + + print() + except Exception as e: + # Here you can log the exception if needed + print(f"An error occurred: {e}") + # Optionally, re-raise the exception if you want it to propagate + raise + finally: + # This block ensures that the driver is closed regardless of an exception + if driver: + driver.quit() + return data diff --git a/wiki/Councils.md b/wiki/Councils.md index e78eb2732d..0ad9844a56 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -178,6 +178,7 @@ This document is still a work in progress, don't worry if your council isn't lis - [StaffordBoroughCouncil](#staffordboroughcouncil) - [Staffordshire Moorlands District Council](#staffordshire-moorlands-district-council) - [Stockport Borough Council](#stockport-borough-council) +- [Stockton On Tees Council](#stockton-on-tees-council) - [Stoke-on-Trent City Council](#stoke-on-trent-city-council) - [Stratford Upon Avon Council](#stratford-upon-avon-council) - [Stroud District Council](#stroud-district-council) @@ -2118,6 +2119,18 @@ Note: Replace XXXXXXXX with UPRN. --- +### Stockton On Tees Council +```commandline +python collect_data.py StocktonOnTeesCouncil https://www.stockton.gov.uk -s -p "XXXX XXX" -n XX -w http://HOST:PORT/ +``` +Additional parameters: +- `-s` - skip get URL +- `-p` - postcode +- `-n` - house number +- `-w` - remote Selenium web driver URL (required for Home Assistant) + +--- + ### Stoke-on-Trent City Council ```commandline python collect_data.py StokeOnTrentCityCouncil https://www.stoke.gov.uk/jadu/custom/webserviceLookUps/BarTecWebServices_missed_bin_calendar.php?UPRN=XXXXXXXXXX