From bf5aa1563aa37b00e778fad3f5768e8756d9247c Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Thu, 16 Feb 2023 20:34:47 +0000 Subject: [PATCH 1/2] Add support for Torbay Council --- .../council_schemas/TorbayCouncil.schema | 39 ++++++++++++ .../features/validate_council_outputs.feature | 1 + uk_bin_collection/tests/input.json | 4 ++ .../tests/outputs/TorbayCouncil.json | 60 +++++++++++++++++++ .../councils/TorbayCouncil.py | 48 +++++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 uk_bin_collection/tests/council_schemas/TorbayCouncil.schema create mode 100644 uk_bin_collection/tests/outputs/TorbayCouncil.json create mode 100644 uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py diff --git a/uk_bin_collection/tests/council_schemas/TorbayCouncil.schema b/uk_bin_collection/tests/council_schemas/TorbayCouncil.schema new file mode 100644 index 0000000000..6f92551be5 --- /dev/null +++ b/uk_bin_collection/tests/council_schemas/TorbayCouncil.schema @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/Welcome10", + "definitions": { + "Welcome10": { + "type": "object", + "additionalProperties": false, + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/definitions/Bin" + } + } + }, + "required": [ + "bins" + ], + "title": "Welcome10" + }, + "Bin": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "collectionDate": { + "type": "string" + } + }, + "required": [ + "collectionDate", + "type" + ], + "title": "Bin" + } + } +} diff --git a/uk_bin_collection/tests/features/validate_council_outputs.feature b/uk_bin_collection/tests/features/validate_council_outputs.feature index fd1d03861a..e5f46e02c2 100644 --- a/uk_bin_collection/tests/features/validate_council_outputs.feature +++ b/uk_bin_collection/tests/features/validate_council_outputs.feature @@ -42,6 +42,7 @@ Feature: Test each council output matches expected results in /outputs | StockportBoroughCouncil | | TamesideMBCouncil | | TonbridgeAndMallingBC | + | TorbayCouncil | | TorridgeDistrictCouncil | | ValeofGlamorganCouncil | | WarwickDistrictCouncil | diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index e6c5c980aa..4355a55c77 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -111,6 +111,10 @@ "postcode": "ME19 4JS", "uprn": "10002914589" }, + "TorbayCouncil": { + "url": "https://www.torbay.gov.uk/recycling/bin-collections/", + "uprn": "10093139404" + }, "TorridgeDistrictCouncil": "10091078762", "ValeofGlamorganCouncil": { "url": "https://www.valeofglamorgan.gov.uk/en/living/Recycling-and-Waste/", diff --git a/uk_bin_collection/tests/outputs/TorbayCouncil.json b/uk_bin_collection/tests/outputs/TorbayCouncil.json new file mode 100644 index 0000000000..544d680a6a --- /dev/null +++ b/uk_bin_collection/tests/outputs/TorbayCouncil.json @@ -0,0 +1,60 @@ +{ + "bins": [ + { + "type": "Food Caddy", + "collectionDate": "22/02/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "22/02/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "22/02/2023" + }, + { + "type": "Residual Bin", + "collectionDate": "22/02/2023" + }, + { + "type": "Food Caddy", + "collectionDate": "01/03/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "01/03/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "01/03/2023" + }, + { + "type": "Food Caddy", + "collectionDate": "08/03/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "08/03/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "08/03/2023" + }, + { + "type": "Residual Bin", + "collectionDate": "08/03/2023" + }, + { + "type": "Food Caddy", + "collectionDate": "15/03/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "15/03/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "15/03/2023" + } + ] +} \ No newline at end of file diff --git a/uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py b/uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py new file mode 100644 index 0000000000..a29e64f8fc --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py @@ -0,0 +1,48 @@ +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + uprn = kwargs.get("uprn") + check_uprn(uprn) + + headers = { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8", + "Connection": "keep-alive", + "Host": "online.torbay.gov.uk", + "Origin": "https://www.torbay.gov.uk", + "Referer": "https://www.torbay.gov.uk/", + "sec-ch-ua": "\"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + requests.packages.urllib3.disable_warnings() + response = requests.get(f"https://online.torbay.gov.uk/services.bartec/collections?uprn={uprn}", headers=headers) + if response.status_code != 200: + raise ValueError("No bin data found for provided UPRN.") + json_data = json.loads(response.text) + + data = {"bins": []} + for c in json_data: + dict_data = { + "type": c["Service"].replace("Empty ", "").strip(), + "collectionDate": datetime.strptime( + c["NextCollection"].strip(), "%d %B %Y" + ).strftime(date_format), + } + data["bins"].append(dict_data) + + return data From c39dfbc20688cafd0e891ebeacb9ec9f00babe1d Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Thu, 16 Feb 2023 20:34:47 +0000 Subject: [PATCH 2/2] Add support for Torbay Council --- .../council_schemas/TorbayCouncil.schema | 39 ++++++++++++ .../features/validate_council_outputs.feature | 1 + uk_bin_collection/tests/input.json | 4 ++ .../tests/outputs/TorbayCouncil.json | 60 +++++++++++++++++++ .../councils/TorbayCouncil.py | 48 +++++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 uk_bin_collection/tests/council_schemas/TorbayCouncil.schema create mode 100644 uk_bin_collection/tests/outputs/TorbayCouncil.json create mode 100644 uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py diff --git a/uk_bin_collection/tests/council_schemas/TorbayCouncil.schema b/uk_bin_collection/tests/council_schemas/TorbayCouncil.schema new file mode 100644 index 0000000000..6f92551be5 --- /dev/null +++ b/uk_bin_collection/tests/council_schemas/TorbayCouncil.schema @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/Welcome10", + "definitions": { + "Welcome10": { + "type": "object", + "additionalProperties": false, + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/definitions/Bin" + } + } + }, + "required": [ + "bins" + ], + "title": "Welcome10" + }, + "Bin": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "collectionDate": { + "type": "string" + } + }, + "required": [ + "collectionDate", + "type" + ], + "title": "Bin" + } + } +} diff --git a/uk_bin_collection/tests/features/validate_council_outputs.feature b/uk_bin_collection/tests/features/validate_council_outputs.feature index fd1d03861a..e5f46e02c2 100644 --- a/uk_bin_collection/tests/features/validate_council_outputs.feature +++ b/uk_bin_collection/tests/features/validate_council_outputs.feature @@ -42,6 +42,7 @@ Feature: Test each council output matches expected results in /outputs | StockportBoroughCouncil | | TamesideMBCouncil | | TonbridgeAndMallingBC | + | TorbayCouncil | | TorridgeDistrictCouncil | | ValeofGlamorganCouncil | | WarwickDistrictCouncil | diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index e6c5c980aa..ccd8ecf141 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -111,6 +111,10 @@ "postcode": "ME19 4JS", "uprn": "10002914589" }, + "TorbayCouncil": { + "url": "https://www.torbay.gov.uk/recycling/bin-collections/", + "uprn": "100041053198" + }, "TorridgeDistrictCouncil": "10091078762", "ValeofGlamorganCouncil": { "url": "https://www.valeofglamorgan.gov.uk/en/living/Recycling-and-Waste/", diff --git a/uk_bin_collection/tests/outputs/TorbayCouncil.json b/uk_bin_collection/tests/outputs/TorbayCouncil.json new file mode 100644 index 0000000000..544d680a6a --- /dev/null +++ b/uk_bin_collection/tests/outputs/TorbayCouncil.json @@ -0,0 +1,60 @@ +{ + "bins": [ + { + "type": "Food Caddy", + "collectionDate": "22/02/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "22/02/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "22/02/2023" + }, + { + "type": "Residual Bin", + "collectionDate": "22/02/2023" + }, + { + "type": "Food Caddy", + "collectionDate": "01/03/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "01/03/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "01/03/2023" + }, + { + "type": "Food Caddy", + "collectionDate": "08/03/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "08/03/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "08/03/2023" + }, + { + "type": "Residual Bin", + "collectionDate": "08/03/2023" + }, + { + "type": "Food Caddy", + "collectionDate": "15/03/2023" + }, + { + "type": "Paper Bag", + "collectionDate": "15/03/2023" + }, + { + "type": "Recycling Box", + "collectionDate": "15/03/2023" + } + ] +} \ No newline at end of file diff --git a/uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py b/uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py new file mode 100644 index 0000000000..a29e64f8fc --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/TorbayCouncil.py @@ -0,0 +1,48 @@ +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + uprn = kwargs.get("uprn") + check_uprn(uprn) + + headers = { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8", + "Connection": "keep-alive", + "Host": "online.torbay.gov.uk", + "Origin": "https://www.torbay.gov.uk", + "Referer": "https://www.torbay.gov.uk/", + "sec-ch-ua": "\"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" + } + requests.packages.urllib3.disable_warnings() + response = requests.get(f"https://online.torbay.gov.uk/services.bartec/collections?uprn={uprn}", headers=headers) + if response.status_code != 200: + raise ValueError("No bin data found for provided UPRN.") + json_data = json.loads(response.text) + + data = {"bins": []} + for c in json_data: + dict_data = { + "type": c["Service"].replace("Empty ", "").strip(), + "collectionDate": datetime.strptime( + c["NextCollection"].strip(), "%d %B %Y" + ).strftime(date_format), + } + data["bins"].append(dict_data) + + return data