diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index fa1afb7fad..fb5204eb3e 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -6,6 +6,12 @@ "wiki_name": "Aberdeenshire Council", "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." }, + "AberdeenCityCouncil": { + "url": "https://www.aberdeencity.gov.uk", + "uprn": "9051156186", + "wiki_name": "Aberdeen City Council", + "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." + }, "AdurAndWorthingCouncils": { "url": "https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=100061878829", "wiki_command_url_override": "https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=XXXXXXXX", @@ -213,6 +219,14 @@ "wiki_name": "Bradford MDC", "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Postcode isn't parsed by this script, but you can pass it in double quotes." }, + "BraintreeDistrictCouncil": { + "postcode": "CO5 9BD", + "skip_get_url": true, + "uprn": "10006930172", + "url": "https://www.braintree.gov.uk/", + "wiki_name": "Braintree District Council", + "wiki_note": "Provide your UPRN and postcode. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN." + }, "BrecklandCouncil": { "url": "https://www.breckland.gov.uk", "wiki_command_url_override": "https://www.breckland.gov.uk", @@ -276,6 +290,12 @@ "wiki_name": "Buckinghamshire Council (Chiltern, South Bucks, Wycombe)", "wiki_note": "Pass the house name/number and postcode in their respective arguments, both wrapped in quotes." }, + "BurnleyBoroughCouncil": { + "uprn": "100010347165", + "url": "https://www.burnley.gov.uk", + "wiki_name": "Burnley Borough Council", + "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search)." + }, "BuryCouncil": { "house_number": "3", "postcode": "M26 3XY", @@ -597,6 +617,14 @@ "wiki_name": "Eastleigh Borough Council", "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search)." }, + "EdinburghCityCouncil": { + "skip_get_url": true, + "house_number": "Tuesday", + "postcode": "Week 1", + "url": "https://www.edinburgh.gov.uk", + "wiki_name": "Edinburgh City Council", + "wiki_note": "Use the House Number field to pass the DAY of the week for your collections. Monday/Tuesday/Wednesday/Thursday/Friday. Use the 'postcode' field to pass the WEEK for your collection. [Week 1/Week 2]" + }, "ElmbridgeBoroughCouncil": { "url": "https://www.elmbridge.gov.uk", "wiki_command_url_override": "https://www.elmbridge.gov.uk", @@ -632,6 +660,12 @@ "wiki_name": "Erewash Borough Council", "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search)." }, + "ExeterCityCouncil": { + "uprn": "100040212270", + "url": "https://www.exeter.gov.uk", + "wiki_name": "Exeter City Council", + "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search)." + }, "FalkirkCouncil": { "url": "https://www.falkirk.gov.uk", "wiki_command_url_override": "https://www.falkirk.gov.uk", diff --git a/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py new file mode 100644 index 0000000000..a69b3ab16b --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py @@ -0,0 +1,122 @@ +import time + +import requests + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + + user_uprn = kwargs.get("uprn") + check_uprn(user_uprn) + bindata = {"bins": []} + + SESSION_URL = "https://integration.aberdeencity.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fintegration.aberdeencity.gov.uk%252Fservice%252Fbin_collection_calendar___view&hostname=integration.aberdeencity.gov.uk&withCredentials=true" + + API_URL = "https://integration.aberdeencity.gov.uk/apibroker/runLookup" + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "Mozilla/5.0", + "X-Requested-With": "XMLHttpRequest", + "Referer": "https://integration.aberdeencity.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=", + } + s = requests.session() + r = s.get(SESSION_URL) + r.raise_for_status() + session_data = r.json() + sid = session_data["auth-session"] + params = { + "id": "583c08ffc47fe", + "repeat_against": "", + "noRetry": "true", + "getOnlyTokens": "undefined", + "log_id": "", + "app_name": "AF-Renderer::Self", + # unix_timestamp + "_": str(int(time.time() * 1000)), + "sid": sid, + } + + r = s.post(API_URL, headers=headers, params=params) + r.raise_for_status() + + data = r.json() + rows_data = data["integration"]["transformed"]["rows_data"]["0"] + if not isinstance(rows_data, dict): + raise ValueError("Invalid data returned from API") + token = rows_data["token"] + + data = { + "formValues": { + "Section 1": { + "nauprn": { + "value": user_uprn, + }, + "token": { + "value": token, + }, + "mindate": { + "value": datetime.now().strftime("%Y-%m-%d"), + }, + "maxdate": { + "value": (datetime.now() + timedelta(days=30)).strftime( + "%Y-%m-%d" + ), + }, + }, + }, + } + + params = { + "id": "5a3141caf4016", + "repeat_against": "", + "noRetry": "true", + "getOnlyTokens": "undefined", + "log_id": "", + "app_name": "AF-Renderer::Self", + # unix_timestamp + "_": str(int(time.time() * 1000)), + "sid": sid, + } + + r = s.post(API_URL, json=data, headers=headers, params=params) + r.raise_for_status() + + data = r.json() + rows_data = data["integration"]["transformed"]["rows_data"]["0"] + if not isinstance(rows_data, dict): + raise ValueError("Invalid data returned from API") + + date_pattern = re.compile(r"^(.*?)(Date\d+)$") + count_pattern = re.compile(r"^Count(.*)$") + for key, value in rows_data.items(): + date_match = date_pattern.match(key) + # Match count keys + count_match = count_pattern.match(key) + if count_match: + continue + + # Match date keys + date_match = date_pattern.match(key) + if date_match: + bin_type = date_match.group(1) + dict_data = { + "type": bin_type, + "collectionDate": datetime.strptime(value, "%A %d %B %Y").strftime( + date_format + ), + } + bindata["bins"].append(dict_data) + + return bindata diff --git a/uk_bin_collection/uk_bin_collection/councils/BraintreeDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BraintreeDistrictCouncil.py new file mode 100644 index 0000000000..5834a41759 --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/BraintreeDistrictCouncil.py @@ -0,0 +1,70 @@ +import time + +import requests +from bs4 import BeautifulSoup + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + + user_postcode = kwargs.get("postcode") + user_uprn = kwargs.get("uprn") + check_postcode(user_postcode) + check_uprn(user_uprn) + bindata = {"bins": []} + + URI = "https://www.braintree.gov.uk/xfp/form/554" + + response = requests.get(URI) + soup = BeautifulSoup(response.content, "html.parser") + token = (soup.find("input", {"name": "__token"})).get("value") + + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", + "Referer": "https://www.braintree.gov.uk/xfp/form/554", + } + + form_data = { + "__token": token, + "page": "5730", + "locale": "en_GB", + "qe15dda0155d237d1ea161004d1839e3369ed4831_0_0": user_postcode, + "qe15dda0155d237d1ea161004d1839e3369ed4831_1_0": user_uprn, + "next": "Next", + } + collection_lookup = requests.post(URI, data=form_data, headers=headers) + collection_lookup.raise_for_status() + for results in BeautifulSoup(collection_lookup.text, "html.parser").find_all( + "div", class_="date_display" + ): + collection_info = results.text.strip().split("\n") + collection_type = collection_info[0].strip() + + # Skip if no collection date is found + if len(collection_info) < 2: + continue + + collection_date = collection_info[1].strip() + + dict_data = { + "type": collection_type, + "collectionDate": collection_date, + } + bindata["bins"].append(dict_data) + + bindata["bins"].sort( + key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y") + ) + + return bindata diff --git a/uk_bin_collection/uk_bin_collection/councils/BurnleyBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BurnleyBoroughCouncil.py new file mode 100644 index 0000000000..97283ce09b --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/BurnleyBoroughCouncil.py @@ -0,0 +1,88 @@ +import time + +import requests + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + + user_uprn = kwargs.get("uprn") + check_uprn(user_uprn) + bindata = {"bins": []} + + SESSION_URL = "https://your.burnley.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fyour.burnley.gov.uk%252Fen%252FAchieveForms%252F%253Fform_uri%253Dsandbox-publish%253A%252F%252FAF-Process-b41dcd03-9a98-41be-93ba-6c172ba9f80c%252FAF-Stage-edb97458-fc4d-4316-b6e0-85598ec7fce8%252Fdefinition.json%2526redirectlink%253D%25252Fen%2526cancelRedirectLink%253D%25252Fen%2526consentMessage%253Dyes&hostname=your.burnley.gov.uk&withCredentials=true" + + API_URL = "https://your.burnley.gov.uk/apibroker/runLookup" + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "Mozilla/5.0", + "X-Requested-With": "XMLHttpRequest", + "Referer": "https://your.burnley.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=", + } + s = requests.session() + r = s.get(SESSION_URL) + r.raise_for_status() + session_data = r.json() + sid = session_data["auth-session"] + + data = { + "formValues": { + "Section 1": { + "case_uprn1": { + "value": user_uprn, + } + }, + }, + } + + params = { + "id": "607fe757df87c", + "repeat_against": "", + "noRetry": "false", + "getOnlyTokens": "undefined", + "log_id": "", + "app_name": "AF-Renderer::Self", + # unix_timestamp + "_": str(int(time.time() * 1000)), + "sid": sid, + } + + r = s.post(API_URL, json=data, headers=headers, params=params) + r.raise_for_status() + + data = r.json() + rows_data = data["integration"]["transformed"]["rows_data"] + if not isinstance(rows_data, dict): + raise ValueError("Invalid data returned from API") + + current_year = (datetime.now()).year + for key, value in rows_data.items(): + bin_type = value["display"].split(" - ")[0] + collection_date = datetime.strptime( + value["display"].split(" - ")[1], "%A %d %B" + ) + + if collection_date.month == 1: + collection_date = collection_date.replace(year=current_year + 1) + else: + collection_date = collection_date.replace(year=current_year) + + dict_data = { + "type": bin_type, + "collectionDate": collection_date.strftime(date_format), + } + bindata["bins"].append(dict_data) + + return bindata diff --git a/uk_bin_collection/uk_bin_collection/councils/EdinburghCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EdinburghCityCouncil.py new file mode 100644 index 0000000000..2212f28695 --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/EdinburghCityCouncil.py @@ -0,0 +1,98 @@ +import re +import time + +import requests +from bs4 import BeautifulSoup +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.wait import WebDriverWait + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + + collection_day = kwargs.get("paon") + collection_week = kwargs.get("postcode") + bindata = {"bins": []} + + days_of_week = [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + "Sunday", + ] + + collection_weeks = ["Week 1", "Week 2"] + collection_week = collection_weeks.index(collection_week) + + offset_days = days_of_week.index(collection_day) + + if collection_week == 0: + recyclingstartDate = datetime(2024, 11, 4) + glassstartDate = datetime(2024, 11, 4) + refusestartDate = datetime(2024, 11, 11) + elif collection_week == 1: + recyclingstartDate = datetime(2024, 11, 11) + glassstartDate = datetime(2024, 11, 11) + refusestartDate = datetime(2024, 11, 4) + + refuse_dates = get_dates_every_x_days(refusestartDate, 14, 28) + glass_dates = get_dates_every_x_days(glassstartDate, 14, 28) + recycling_dates = get_dates_every_x_days(recyclingstartDate, 14, 28) + + for refuseDate in refuse_dates: + + collection_date = ( + datetime.strptime(refuseDate, "%d/%m/%Y") + timedelta(days=offset_days) + ).strftime("%d/%m/%Y") + + dict_data = { + "type": "Grey Bin", + "collectionDate": collection_date, + } + bindata["bins"].append(dict_data) + + for recyclingDate in recycling_dates: + + collection_date = ( + datetime.strptime(recyclingDate, "%d/%m/%Y") + + timedelta(days=offset_days) + ).strftime("%d/%m/%Y") + + dict_data = { + "type": "Green Bin", + "collectionDate": collection_date, + } + bindata["bins"].append(dict_data) + + for glassDate in glass_dates: + + collection_date = ( + datetime.strptime(glassDate, "%d/%m/%Y") + timedelta(days=offset_days) + ).strftime("%d/%m/%Y") + + dict_data = { + "type": "Glass Box", + "collectionDate": collection_date, + } + bindata["bins"].append(dict_data) + + bindata["bins"].sort( + key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y") + ) + + return bindata diff --git a/uk_bin_collection/uk_bin_collection/councils/ExeterCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ExeterCityCouncil.py new file mode 100644 index 0000000000..c35c49676c --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/ExeterCityCouncil.py @@ -0,0 +1,52 @@ +import time + +import requests +from bs4 import BeautifulSoup + +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + + user_uprn = kwargs.get("uprn") + check_uprn(user_uprn) + bindata = {"bins": []} + + URI = f"https://exeter.gov.uk/repositories/hidden-pages/address-finder/?qsource=UPRN&qtype=bins&term={user_uprn}" + + response = requests.get(URI) + response.raise_for_status() + + data = response.json() + + soup = BeautifulSoup(data[0]["Results"], "html.parser") + soup.prettify() + + # Extract bin schedule + for section in soup.find_all("h2"): + bin_type = section.text.strip() + collection_date = section.find_next("h3").text.strip() + + dict_data = { + "type": bin_type, + "collectionDate": datetime.strptime( + remove_ordinal_indicator_from_date_string(collection_date), + "%A, %d %B %Y", + ).strftime(date_format), + } + bindata["bins"].append(dict_data) + + bindata["bins"].sort( + key=lambda x: datetime.strptime(x.get("collectionDate"), date_format) + ) + + return bindata diff --git a/wiki/Councils.md b/wiki/Councils.md index 3f6919219c..4dddfe2247 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -11,6 +11,7 @@ This document is still a work in progress, don't worry if your council isn't lis ## Contents - [Aberdeenshire Council](#aberdeenshire-council) +- [Aberdeen City Council](#aberdeen-city-council) - [Adur and Worthing Councils](#adur-and-worthing-councils) - [Antrim & Newtonabbey Council](#antrim-&-newtonabbey-council) - [Ards and North Down Council](#ards-and-north-down-council) @@ -37,6 +38,7 @@ This document is still a work in progress, don't worry if your council isn't lis - [Bolton Council](#bolton-council) - [Bracknell Forest Council](#bracknell-forest-council) - [Bradford MDC](#bradford-mdc) +- [Braintree District Council](#braintree-district-council) - [Breckland Council](#breckland-council) - [Brighton and Hove City Council](#brighton-and-hove-city-council) - [Bristol City Council](#bristol-city-council) @@ -45,6 +47,7 @@ This document is still a work in progress, don't worry if your council isn't lis - [Broxbourne Council](#broxbourne-council) - [Broxtowe Borough Council](#broxtowe-borough-council) - [Buckinghamshire Council (Chiltern, South Bucks, Wycombe)](#buckinghamshire-council-(chiltern,-south-bucks,-wycombe)) +- [Burnley Borough Council](#burnley-borough-council) - [Bury Council](#bury-council) - [Calderdale Council](#calderdale-council) - [Cannock Chase District Council](#cannock-chase-district-council) @@ -86,11 +89,13 @@ This document is still a work in progress, don't worry if your council isn't lis - [East Riding Council](#east-riding-council) - [East Suffolk Council](#east-suffolk-council) - [Eastleigh Borough Council](#eastleigh-borough-council) +- [Edinburgh City Council](#edinburgh-city-council) - [Elmbridge Borough Council](#elmbridge-borough-council) - [Enfield Council](#enfield-council) - [Environment First](#environment-first) - [Epping Forest District Council](#epping-forest-district-council) - [Erewash Borough Council](#erewash-borough-council) +- [Exeter City Council](#exeter-city-council) - [Falkirk Council](#falkirk-council) - [Fareham Borough Council](#fareham-borough-council) - [Fenland District Council](#fenland-district-council) @@ -274,6 +279,17 @@ Note: You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/searc --- +### Aberdeen City Council +```commandline +python collect_data.py AberdeenCityCouncil https://www.aberdeencity.gov.uk -u XXXXXXXX +``` +Additional parameters: +- `-u` - UPRN + +Note: You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. + +--- + ### Adur and Worthing Councils ```commandline python collect_data.py AdurAndWorthingCouncils https://www.adur-worthing.gov.uk/bin-day/?brlu-selected-address=XXXXXXXX @@ -593,6 +609,19 @@ Note: To get the UPRN, you will need to use [FindMyAddress](https://www.findmyad --- +### Braintree District Council +```commandline +python collect_data.py BraintreeDistrictCouncil https://www.braintree.gov.uk/ -s -u XXXXXXXX -p "XXXX XXX" +``` +Additional parameters: +- `-s` - skip get URL +- `-u` - UPRN +- `-p` - postcode + +Note: Provide your UPRN and postcode. Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN. + +--- + ### Breckland Council ```commandline python collect_data.py BrecklandCouncil https://www.breckland.gov.uk -u XXXXXXXX @@ -693,6 +722,17 @@ Note: Pass the house name/number and postcode in their respective arguments, bot --- +### Burnley Borough Council +```commandline +python collect_data.py BurnleyBoroughCouncil https://www.burnley.gov.uk -u XXXXXXXX +``` +Additional parameters: +- `-u` - UPRN + +Note: Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search). + +--- + ### Bury Council ```commandline python collect_data.py BuryCouncil https://www.bury.gov.uk/waste-and-recycling/bin-collection-days-and-alerts -s -p "XXXX XXX" -n XX @@ -1189,6 +1229,19 @@ Note: Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyadd --- +### Edinburgh City Council +```commandline +python collect_data.py EdinburghCityCouncil https://www.edinburgh.gov.uk -s -p "XXXX XXX" -n XX +``` +Additional parameters: +- `-s` - skip get URL +- `-p` - postcode +- `-n` - house number + +Note: Use the House Number field to pass the DAY of the week for your collections. Monday/Tuesday/Wednesday/Thursday/Friday. Use the 'postcode' field to pass the WEEK for your collection. [Week 1/Week 2] + +--- + ### Elmbridge Borough Council ```commandline python collect_data.py ElmbridgeBoroughCouncil https://www.elmbridge.gov.uk -u XXXXXXXX @@ -1246,6 +1299,17 @@ Note: Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyadd --- +### Exeter City Council +```commandline +python collect_data.py ExeterCityCouncil https://www.exeter.gov.uk -u XXXXXXXX +``` +Additional parameters: +- `-u` - UPRN + +Note: Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search). + +--- + ### Falkirk Council ```commandline python collect_data.py FalkirkCouncil https://www.falkirk.gov.uk -u XXXXXXXX