From f557d946ac44978feea06bd3aa2e09a682af2b97 Mon Sep 17 00:00:00 2001 From: Wiki GitHub Action Date: Wed, 18 Oct 2023 21:11:31 +0000 Subject: [PATCH 1/4] docs: Update Councils.md from input.json --- wiki/Councils.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/wiki/Councils.md b/wiki/Councils.md index e13112d7b6..48cb4fe30e 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -25,7 +25,7 @@ This document is still a work in progress, don't worry if your council isn't lis - [Charnwood Borough Council](#charnwood-borough-council) - [Chelmsford City Council](#chelmsford-city-council) - [Cheshire East Council](#cheshire-east-council) -- [Chilterns](#chilterns) +- [Buckinghamshire Council (Chiltern, South Bucks, Wycombe)](#buckinghamshire-council-(chiltern,-south-bucks,-wycombe)) - [Crawley Borough Council](#crawley-borough-council) - [Croydon Council](#croydon-council) - [Derbyshire Dales District Council](#derbyshire-dales-district-council) @@ -81,6 +81,7 @@ This document is still a work in progress, don't worry if your council isn't lis - [South Tyneside Council](#south-tyneside-council) - [St Helens Borough Council](#st-helens-borough-council) - [Stockport Borough Council](#stockport-borough-council) +- [Stratford Upon Avon Council](#stratford-upon-avon-council) - [Swale Borough Council](#swale-borough-council) - [Tameside Metropolitan Borough Council](#tameside-metropolitan-borough-council) - [Tonbridge and Malling Borough Council](#tonbridge-and-malling-borough-council) @@ -258,9 +259,9 @@ Use the form [here](https://online.cheshireeast.gov.uk/mycollectionday/) to find --- -### Chilterns +### Buckinghamshire Council (Chiltern, South Bucks, Wycombe) ```commandline -python collect_data.py Chilterns https://chiltern.gov.uk/collection-dates -s -p "XXXX XXX" -n XX +python collect_data.py BuckinghamshireCouncil https://chiltern.gov.uk/collection-dates -s -p "XXXX XXX" -n XX ``` Additional parameters: - `-s` - skip get URL @@ -273,12 +274,12 @@ Note: Pass the name of the street with the house number parameter, wrapped in do ### Crawley Borough Council ```commandline -python collect_data.py CrawleyBoroughCouncil https://my.crawley.gov.uk/ -s -u XXXXXXXX -usrn XXXXXXXX +python collect_data.py CrawleyBoroughCouncil https://my.crawley.gov.uk/ -s -u XXXXXXXX -n XX ``` Additional parameters: - `-s` - skip get URL - `-u` - UPRN -- `-us` - USRN +- `-n` - house number Note: Crawley needs to be passed both a UPRN and a USRN to work. Find these on [FindMyAddress](https://www.findmyaddress.co.uk/search) or [FindMyStreet](https://www.findmystreet.co.uk/map). @@ -847,6 +848,16 @@ Note: Replace XXXXXXXX with UPRN. --- +### Stratford Upon Avon Council +```commandline +python collect_data.py StratfordUponAvonCouncil https://www.stratford.gov.uk/waste-recycling/when-we-collect.cfm/part/calendar -s -u XXXXXXXX +``` +Additional parameters: +- `-s` - skip get URL +- `-u` - UPRN + +--- + ### Swale Borough Council ```commandline python collect_data.py SwaleBoroughCouncil https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days -s -u XXXXXXXX -p "XXXX XXX" From d8d804fc09c8518514998ffb93a6492f19ad4611 Mon Sep 17 00:00:00 2001 From: David Park Date: Wed, 18 Oct 2023 22:50:44 +0100 Subject: [PATCH 2/4] feat: Add support for Bury Council (#265) --- .../uk_bin_collection/councils/BuryCouncil.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 uk_bin_collection/uk_bin_collection/councils/BuryCouncil.py diff --git a/uk_bin_collection/uk_bin_collection/councils/BuryCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BuryCouncil.py new file mode 100644 index 0000000000..b507057135 --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/BuryCouncil.py @@ -0,0 +1,58 @@ +from bs4 import BeautifulSoup +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import \ + AbstractGetBinDataClass + + +# import the wonderful Beautiful Soup and the URL grabber +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + # Make a BS4 object + collections = [] + data = {"bins": []} + + # Get and check postcode and PAON + postcode = kwargs.get('postcode') + paon = kwargs.get('paon') + check_postcode(postcode) + check_paon(paon) + + # Make API call to get property info using postcode + addr_response = requests.get( + f'https://www.bury.gov.uk/app-services/getProperties?postcode={postcode.replace(" ", "")}') + if addr_response.status_code != 200: + raise ConnectionAbortedError("Issue encountered getting addresses.") + address_json = json.loads(addr_response.text)['response'] + + # This makes addr the next item that has the house number. Since these are ordered by house number, a single + # number like 3 wouldn't return 33 + addr = next(item for item in address_json if paon in item["addressLine1"]) + + # Make API call to get bin data using property ID + response = requests.get(f'https://www.bury.gov.uk/app-services/getPropertyById?id={addr.get("id")}') + if response.status_code != 200: + raise ConnectionAbortedError("Issue encountered getting bin data.") + bin_list = json.loads(response.text)['response']['bins'] + + # The JSON actually returns the next collections and a large calendar. But I opted just for the next dates. + for bin_colour, collection_data in bin_list.items(): + bin_type = bin_colour + bin_date = datetime.strptime(remove_ordinal_indicator_from_date_string(collection_data.get('nextCollection')), "%A %d %B %Y") + collections.append((bin_type, bin_date)) + + # Dates are ordered correctly - soonest first + ordered_data = sorted(collections, key=lambda x: x[1]) + for item in ordered_data: + dict_data = { + "type": item[0], + "collectionDate": item[1].strftime(date_format), + } + data["bins"].append(dict_data) + + return data From ada6ccce37b4d7680c193d71c1bfff97458f6c46 Mon Sep 17 00:00:00 2001 From: David Park Date: Wed, 18 Oct 2023 22:55:36 +0100 Subject: [PATCH 3/4] feat: Add support for Bury Council (#265) --- .../tests/council_schemas/BuryCouncil.schema | 39 +++++++++++++++++++ .../features/validate_council_outputs.feature | 1 + uk_bin_collection/tests/input.json | 8 ++++ .../tests/outputs/BuryCouncil.json | 20 ++++++++++ 4 files changed, 68 insertions(+) create mode 100644 uk_bin_collection/tests/council_schemas/BuryCouncil.schema create mode 100644 uk_bin_collection/tests/outputs/BuryCouncil.json diff --git a/uk_bin_collection/tests/council_schemas/BuryCouncil.schema b/uk_bin_collection/tests/council_schemas/BuryCouncil.schema new file mode 100644 index 0000000000..31805dad36 --- /dev/null +++ b/uk_bin_collection/tests/council_schemas/BuryCouncil.schema @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/Welcome8", + "definitions": { + "Welcome8": { + "type": "object", + "additionalProperties": false, + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/definitions/Bin" + } + } + }, + "required": [ + "bins" + ], + "title": "Welcome8" + }, + "Bin": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "collectionDate": { + "type": "string" + } + }, + "required": [ + "collectionDate", + "type" + ], + "title": "Bin" + } + } +} diff --git a/uk_bin_collection/tests/features/validate_council_outputs.feature b/uk_bin_collection/tests/features/validate_council_outputs.feature index a6bf04e72b..de14bbcf8a 100644 --- a/uk_bin_collection/tests/features/validate_council_outputs.feature +++ b/uk_bin_collection/tests/features/validate_council_outputs.feature @@ -19,6 +19,7 @@ Feature: Test each council output matches expected results in /outputs | BromleyBoroughCouncil | | BroxtoweBoroughCouncil | | BuckinghamshireCouncil | + | BuryCouncil | | CardiffCouncil | | CastlepointDistrictCouncil | | CharnwoodBoroughCouncil | diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index f9b5e1cc18..09e253986c 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -102,6 +102,14 @@ "wiki_name": "Buckinghamshire Council (Chiltern, South Bucks, Wycombe)", "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" }, + "BuryCouncil": { + "SKIP_GET_URL": "SKIP_GET_URL", + "house_number": "3", + "postcode": "M26 3XY", + "url": "https://www.bury.gov.uk/waste-and-recycling/bin-collection-days-and-alerts", + "wiki_name": "Bury Council", + "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes." + }, "CrawleyBoroughCouncil": { "SKIP_GET_URL": "SKIP_GET_URL", "uprn": "100061785321", diff --git a/uk_bin_collection/tests/outputs/BuryCouncil.json b/uk_bin_collection/tests/outputs/BuryCouncil.json new file mode 100644 index 0000000000..9d1d9b949d --- /dev/null +++ b/uk_bin_collection/tests/outputs/BuryCouncil.json @@ -0,0 +1,20 @@ +{ + "bins": [ + { + "type": "green", + "collectionDate": "19/10/2023" + }, + { + "type": "brown", + "collectionDate": "26/10/2023" + }, + { + "type": "blue", + "collectionDate": "26/10/2023" + }, + { + "type": "grey", + "collectionDate": "02/11/2023" + } + ] +} \ No newline at end of file From 48453485dc88f1c655edf4766b29060427963330 Mon Sep 17 00:00:00 2001 From: David Park Date: Wed, 18 Oct 2023 22:57:59 +0100 Subject: [PATCH 4/4] fix: correctly align input.json --- uk_bin_collection/tests/input.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 09e253986c..86befd4ec6 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -103,12 +103,12 @@ "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" }, "BuryCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "3", - "postcode": "M26 3XY", - "url": "https://www.bury.gov.uk/waste-and-recycling/bin-collection-days-and-alerts", - "wiki_name": "Bury Council", - "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes." + "SKIP_GET_URL": "SKIP_GET_URL", + "house_number": "3", + "postcode": "M26 3XY", + "url": "https://www.bury.gov.uk/waste-and-recycling/bin-collection-days-and-alerts", + "wiki_name": "Bury Council", + "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes." }, "CrawleyBoroughCouncil": { "SKIP_GET_URL": "SKIP_GET_URL",