From 691efa6615795a173b97106fd503f23e1a8ac740 Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Thu, 2 Nov 2023 21:12:29 +0000 Subject: [PATCH 1/5] feat: Add remote Selenium web driver support --- .github/workflows/behave.yml | 5 + .../uk_bin_collection/config_flow.py | 4 + .../uk_bin_collection/strings.json | 1 + uk_bin_collection/tests/input.json | 1322 +++++++++-------- .../tests/step_defs/test_validate_council.py | 6 +- .../tests/test_common_functions.py | 37 +- .../uk_bin_collection/collect_data.py | 8 + uk_bin_collection/uk_bin_collection/common.py | 50 +- .../councils/BroxtoweBoroughCouncil.py | 3 +- .../councils/BuckinghamshireCouncil.py | 3 +- .../DerbyshireDalesDistrictCouncil.py | 3 +- .../councils/EastLindseyDistrictCouncil.py | 3 +- .../councils/EastSuffolkCouncil.py | 3 +- .../councils/GatesheadCouncil.py | 3 +- .../councils/HighPeakCouncil.py | 3 +- .../councils/NeathPortTalbotCouncil.py | 3 +- .../councils/NorthNorfolkDistrictCouncil.py | 3 +- .../councils/NorthumberlandCouncil.py | 3 +- .../councils/PrestonCityCouncil.py | 3 +- .../ReigateAndBansteadBoroughCouncil.py | 3 +- .../councils/RushcliffeBoroughCouncil.py | 3 +- .../StaffordshireMoorlandsDistrictCouncil.py | 3 +- .../councils/WakefieldCityCouncil.py | 2 +- .../councils/WestLothianCouncil.py | 3 +- .../uk_bin_collection/get_bin_data.py | 13 +- wiki/generate_wiki.py | 3 + 26 files changed, 793 insertions(+), 703 deletions(-) diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml index 14a7916788..953161c8ea 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave.yml @@ -17,6 +17,11 @@ on: schedule: - cron: '0 0 * * *' +services: + selenium: + image: selenium/standalone-chrome:latest + options: --shm-size=2gb + jobs: build: runs-on: ubuntu-latest diff --git a/custom_components/uk_bin_collection/config_flow.py b/custom_components/uk_bin_collection/config_flow.py index 50c5207110..dc9f11e140 100644 --- a/custom_components/uk_bin_collection/config_flow.py +++ b/custom_components/uk_bin_collection/config_flow.py @@ -50,6 +50,10 @@ async def get_council_schema(self, council=str) -> vol.Schema: council_schema = council_schema.extend( {vol.Required("usrn", default=""): cv.string} ) + if "web_driver" in self.councils_data[council]: + council_schema = council_schema.extend( + {vol.Required("web_driver", default=""): cv.string} + ) return council_schema async def async_step_user(self, user_input=None): diff --git a/custom_components/uk_bin_collection/strings.json b/custom_components/uk_bin_collection/strings.json index e8c9a4a16d..398a69e244 100644 --- a/custom_components/uk_bin_collection/strings.json +++ b/custom_components/uk_bin_collection/strings.json @@ -18,6 +18,7 @@ "postcode": "Postcode of the address", "number": "House number of the address", "usrn": "USRN (Unique Street Reference Number)", + "web_driver": "URL of the remote Selenium web driver to use", "submit": "Submit" }, "description": "Please refer to your councils [wiki](https://github.com/robbrad/UKBinCollectionData/wiki/Councils) entry for details on what to enter" diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index bf6983330c..c9df4c0e60 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1,655 +1,671 @@ { - "AylesburyValeCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "766252532", - "url": "http://avdcbins.web-labs.co.uk/RefuseApi.asmx", - "wiki_name": "Aylesbury Vale Council (Buckinghamshire)", - "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. Returns all published collections in the past, present, future" - }, - "BCPCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100040810214", - "url": "https://online.bcpcouncil.gov.uk/bindaylookup/", - "wiki_name": "BCP Council" - }, - "BasingstokeCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100060220926", - "url": "https://www.basingstoke.gov.uk/bincollection", - "wiki_name": "Basingstoke Council" - }, - "BathAndNorthEastSomersetCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100120000855", - "url": "https://www.bathnes.gov.uk/webforms/waste/collectionday/", - "wiki_name": "Bath and North East Somerset Council" - }, - "BexleyCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "spamstorage@live.co.uk", - "url": "https://www.bexley.gov.uk/", - "wiki_name": "Bexley Council", - "wiki_note": "In order to use this parser, you will need to sign up to [Bexley's @Home app](https://www.bexley.gov.uk/services/rubbish-and-recycling/bexley-home-recycling-app/about-app) (available for [iOS](https://apps.apple.com/gb/app/home-collection-reminder/id1050703690) and [Android](https://play.google.com/store/apps/details?id=com.contender.athome.android)).\nComplete the setup by entering your email and setting your address with postcode and address line.\nOnce you can see the calendar, you _should_ be good to run the parser.\nJust pass the email you used in quotes in the UPRN parameter.\n" - }, - "BlackburnCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100010733027", - "url": "https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn=100010733027&month=8&year=2022", - "wiki_command_url_override": "https://www.blackburn.gov.uk", - "wiki_name": "Blackburn Council" - }, - "BoltonCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100010886949|21 HEATON AVENUE, BOLTON, BL1 5PQ", - "url": "https://www.bolton.gov.uk/next-bin-collection", - "wiki_name": "Bolton Council", - "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN, and the Bolton Council [website](https://www.bolton.gov.uk/next-bin-collection) to find the full address. See [here](https://github.com/robbrad/UKBinCollectionData/issues/272) for more information." - }, - "BristolCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "137547", - "url": "https://bristolcouncil.powerappsportals.com/completedynamicformunauth/?servicetypeid=7dce896c-b3ba-ea11-a812-000d3a7f1cdc", - "wiki_name": "Bristol City Council" - }, - "BromleyBoroughCouncil": { - "url": "https://recyclingservices.bromley.gov.uk/waste/6087017", - "wiki_command_url_override": "https://recyclingservices.bromley.gov.uk/waste/XXXXXXX", - "wiki_name": "Bromley Borough Council", - "wiki_note": "Follow the instructions [here](https://recyclingservices.bromley.gov.uk/waste) until the \"Your bin days\" page then copy the URL and replace the URL in the command." - }, - "BroxtoweBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "NG16 2LY", - "uprn": "100031325997", - "url": "https://www.broxtowe.gov.uk/", - "wiki_name": "Broxtowe Borough Council" - }, - "BuckinghamshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "HUGHENDEN MANOR, MANOR ROAD, HUGHENDEN VALLEY, HIGH WYCOMBE", - "postcode": "HP14 4LA", - "url": "https://chiltern.gov.uk/collection-dates", - "wiki_name": "Buckinghamshire Council (Chiltern, South Bucks, Wycombe)", - "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" - }, - "BuryCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "3", - "postcode": "M26 3XY", - "url": "https://www.bury.gov.uk/waste-and-recycling/bin-collection-days-and-alerts", - "wiki_name": "Bury Council", - "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes." - }, - "CalderdaleCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "OL14 7EX", - "uprn": "010035034598", - "url": "https://www.calderdale.gov.uk/", - "wiki_name": "Calderdale Council" - }, - "CannockChaseDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "WS15 1JA", - "uprn": "200003095389", - "url": "https://www.cannockchasedc.gov.uk/", - "wiki_name": "Cannock Chase District Council", - "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" - }, - "CardiffCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100100112419", - "url": "https://www.cardiff.gov.uk/ENG/resident/Rubbish-and-recycling/When-are-my-bins-collected/Pages/default.aspx", - "wiki_name": "Cardiff Council" - }, - "CastlepointDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "4525", - "url": "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar", - "wiki_name": "Castlepoint District Council" - }, - "CharnwoodBoroughCouncil": { - "url": "https://my.charnwood.gov.uk/location?put=cbc10070067259&rememberme=0&redirect=%2F", - "wiki_command_url_override": "https://my.charnwood.gov.uk/location?put=cbcXXXXXXXX&rememberme=0&redirect=%2F", - "wiki_name": "Charnwood Borough Council", - "wiki_note": "Replace XXXXXXXX with UPRN keeping \"cbc\" before it." - }, - "ChelmsfordCityCouncil": { - "url": "https://www.chelmsford.gov.uk/myhome/?entry[655e95fb-5f84-4d1b-ab65-878a2f3e3ce4]=14368859", - "wiki_command_url_override": "https://www.chelmsford.gov.uk/myhome/XXXXXX", - "wiki_name": "Chelmsford City Council", - "wiki_note": "Follow the instructions [here](https://www.chelmsford.gov.uk/myhome/) until you get the page listing your \"Address\", \"Ward\" etc then copy the URL and replace the URL in the command." - }, - "CheshireEastCouncil": { - "url": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=100012791226&onelineaddress=3%20COBBLERS%20YARD,%20SK9%207DZ&_=1689413260149", - "wiki_command_url_override": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=XXXXXXXX&onelineaddress=XXXXXXXX&_=1689413260149", - "wiki_name": "Cheshire East Council", - "wiki_note": "Both the UPRN and a one-line address are passed in the URL, which needs to be wrapped in double quotes. The one-line address is made up of the house number, street name and postcode.\nUse the form [here](https://online.cheshireeast.gov.uk/mycollectionday/) to find them, then take the first line and post code and replace all spaces with `%20`." - }, - "ConwyCountyBorough": { - "postcode": "LL30 2DF", - "uprn": "100100429249", - "url": "https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas.asp?ilangid=1&uprn=100100429249", - "wiki_name": "Conwy County Borough Council", - "wiki_note": "Conwy County Borough Council is a straight up uprn in the url eg &uprn=XXXXXXXXXXXXX ." - }, - "CrawleyBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "9701076", - "uprn": "100061785321", - "url": "https://my.crawley.gov.uk/", - "wiki_name": "Crawley Borough Council", - "wiki_note": "Crawley needs to be passed both a UPRN and a USRN to work. Find these on [FindMyAddress](https://www.findmyaddress.co.uk/search) or [FindMyStreet](https://www.findmystreet.co.uk/map)." - }, - "CroydonCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "13", - "postcode": "SE25 5DW", - "url": "https://service.croydon.gov.uk/wasteservices/w/webpage/bin-day-enter-address", - "wiki_name": "Croydon Council" - }, - "DerbyshireDalesDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "DE4 3AS", - "uprn": "10070102161", - "url": "https://www.derbyshiredales.gov.uk/", - "wiki_name": "Derbyshire Dales District Council" - }, - "DoncasterCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100050768956", - "url": "https://www.doncaster.gov.uk/Compass/Entity/Launch/D3/", - "wiki_name": "Doncaster Council" - }, - "DurhamCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "200003218818", - "url": "https://www.durham.gov.uk/bincollections?uprn=", - "wiki_name": "Durham Council" - }, - "EastCambridgeshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "10002597178", - "url": "https://www.eastcambs.gov.uk/", - "wiki_name": "East Cambridgeshire Council" - }, - "EastDevonDC": { - "url": "https://eastdevon.gov.uk/recycling-and-waste/recycling-and-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=010090909915", - "wiki_command_url_override": "https://eastdevon.gov.uk/recycling-and-waste/recycling-and-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=XXXXXXXX", - "wiki_name": "East Devon District Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "EastLindseyDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "Raf Coningsby", - "postcode": "LN4 4SY", - "url": "https://www.e-lindsey.gov.uk/", - "wiki_name": "East Lindsey District Council", - "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" - }, - "NorthNorthamptonshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100031021317", - "url": "https://cms.northnorthants.gov.uk/bin-collection-search/calendarevents/100031021318/2023-10-17/2023-10-01", - "wiki_name": "North Northamptonshire Council" - }, - "EastRidingCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "HU17 8LG", - "url": "https://wasterecyclingapi.eastriding.gov.uk", - "wiki_name": "East Riding Council" - }, - "EastSuffolkCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "IP11 9FJ", - "uprn": "10093544720", - "url": "https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder", - "wiki_name": "East Suffolk Council", - "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" - }, - "EastleighBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100060303535", - "url": "https://www.eastleigh.gov.uk/waste-bins-and-recycling/collection-dates/your-waste-bin-and-recycling-collections?uprn=", - "wiki_name": "Eastleigh Borough Council" - }, - "ErewashBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "10003582028", - "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx", - "wiki_name": "Erewash Borough Council" - }, - "FenlandDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "200002981143", - "url": "https://www.fenland.gov.uk/article/13114/", - "wiki_name": "Fenland District Council" - }, - "GatesheadCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "Bracken Cottage", - "postcode": "NE16 5LQ", - "url": "https://www.gateshead.gov.uk/", - "wiki_name": "Gateshead Council", - "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" - }, - "GlasgowCityCouncil": { - "url": "https://www.glasgow.gov.uk/forms/refuseandrecyclingcalendar/PrintCalendar.aspx?UPRN=906700034497", - "wiki_command_url_override": "https://www.glasgow.gov.uk/forms/refuseandrecyclingcalendar/PrintCalendar.aspx?UPRN=XXXXXXXX", - "wiki_name": "Glasgow City Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "GuildfordCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100061372691", - "url": "https://my.guildford.gov.uk/customers/s/sfsites/aura?r=15&other.BinScheduleDisplayCmp.GetBinSchedules=1", - "wiki_name": "Guildford Council", - "wiki_note": "If the bin day is 'today' then the collectionDate will only show today's date if before 7AM, else the date will be in 'previousCollectionDate'. To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." - }, - "HarrogateBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100050414307", - "url": "https://secure.harrogate.gov.uk/inmyarea", - "wiki_name": "Harrogate Borough Council", - "wiki_note": "Pass the UPRN which can be found at https://secure.harrogate.gov.uk/inmyarea URL doesn't need to be passed." - }, - "HighPeakCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "9 Ellison Street, Glossop", - "postcode": "SK13 8BX", - "url": "https://www.highpeak.gov.uk/findyourbinday", - "wiki_name": "High Peak Council", - "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" - }, - "HuntingdonDistrictCouncil": { - "url": "http://www.huntingdonshire.gov.uk/refuse-calendar/10012048679", - "wiki_command_url_override": "https://www.huntingdonshire.gov.uk/refuse-calendar/XXXXXXXX", - "wiki_name": "Huntingdon District Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "KingstonUponThamesCouncil": { - "url": "https://waste-services.kingston.gov.uk/waste/2701097", - "wiki_command_url_override": "https://waste-services.kingston.gov.uk/waste/XXXXXXX", - "wiki_name": "Kingston Upon Thames Council", - "wiki_note": "Follow the instructions [here](https://waste-services.kingston.gov.uk/waste) until the \"Your bin days\" page then copy the URL and replace the URL in the command." - }, - "LancasterCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "1", - "postcode": "LA1 1RS", - "url": "https://lcc-wrp.whitespacews.com", - "wiki_name": "Lancaster City Council" - }, - "LeedsCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "1", - "postcode": "LS6 2SE", - "url": "https://www.leeds.gov.uk/residents/bins-and-recycling/check-your-bin-day", - "wiki_name": "Leeds City Council" - }, - "LisburnCastlereaghCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "97", - "postcode": "BT28 1JN", - "url": "https://lisburn.isl-fusion.com", - "wiki_name": "Lisburn and Castlereagh City Council" - }, - "LondonBoroughHounslow": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100021577765", - "url": "https://www.hounslow.gov.uk/homepage/86/recycling_and_waste_collection_day_finder", - "wiki_name": "London Borough Hounslow" - }, - "MaldonDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100090557253", - "url": "https://maldon.suez.co.uk/maldon/ServiceSummary", - "wiki_name": "Maldon District Council" - }, - "MalvernHillsDC": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100121348457", - "url": "https://swict.malvernhills.gov.uk/mhdcroundlookup/HandleSearchScreen", - "wiki_name": "Malvern Hills District Council" - }, - "ManchesterCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "000077048077", - "url": "https://www.manchester.gov.uk/bincollections", - "wiki_name": "Manchester City Council" - }, - "MertonCouncil": { - "url": "https://myneighbourhood.merton.gov.uk/Wasteservices/WasteServices.aspx?ID=28186366", - "wiki_command_url_override": "https://myneighbourhood.merton.gov.uk/Wasteservices/WasteServices.aspx?ID=XXXXXXXX", - "wiki_name": "Merton Council", - "wiki_note": "Follow the instructions [here](https://myneighbourhood.merton.gov.uk/Wasteservices/WasteServicesSearch.aspx) until you get the \"Your recycling and rubbish collection days\" page then copy the URL and replace the URL in the command (the Address parameter is optional)." - }, - "MidSussexDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "OAKLANDS", - "postcode": "RH16 1SS", - "url": "https://www.midsussex.gov.uk/waste-recycling/bin-collection/", - "wiki_name": "Mid Sussex District Council", - "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" - }, - "MiltonKeynesCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "Fullers Slade", - "url": "https://www.milton-keynes.gov.uk/waste-and-recycling/collection-days", - "wiki_name": "Milton Keynes City Council", - "wiki_note": "Pass the name of the estate with the UPRN parameter, wrapped in double quotes" - }, - "NeathPortTalbotCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "SA13 3BA", - "uprn": "10023947752", - "url": "https://www.npt.gov.uk", - "wiki_name": "Neath Port Talbot Council" - }, - "NewarkAndSherwoodDC": { - "url": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=200004258529", - "wiki_command_url_override": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX", - "wiki_name": "Newark and Sherwood District Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "NewcastleCityCouncil": { - "url": "https://community.newcastle.gov.uk/my-neighbourhood/ajax/getBinsNew.php?uprn=004510730634", - "wiki_command_url_override": "https://community.newcastle.gov.uk/my-neighbourhood/ajax/getBinsNew.php?uprn=XXXXXXXX", - "wiki_name": "Newcastle City Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "NorthEastLincs": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "11062649", - "url": "https://www.nelincs.gov.uk/refuse-collection-schedule/?view=timeline&uprn=11062649", - "wiki_command_url_override": "https://www.nelincs.gov.uk/refuse-collection-schedule/?view=timeline&uprn=XXXXXXXX", - "wiki_name": "North East Lincolnshire Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "NorthKestevenDistrictCouncil": { - "url": "https://www.n-kesteven.org.uk/bins/display?uprn=100030869513", - "wiki_command_url_override": "https://www.n-kesteven.org.uk/bins/display?uprn=XXXXXXXX", - "wiki_name": "North Kesteven District Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "NorthLanarkshireCouncil": { - "url": "https://www.northlanarkshire.gov.uk/bin-collection-dates/000118016164/48402118", - "wiki_command_url_override": "https://www.northlanarkshire.gov.uk/bin-collection-dates/XXXXXXXXXXX/XXXXXXXXXXX", - "wiki_name": "North Lanarkshire Council", - "wiki_note": "Follow the instructions [here](https://www.northlanarkshire.gov.uk/bin-collection-dates) until you get the \"Next collections\" page then copy the URL and replace the URL in the command." - }, - "NorthLincolnshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100050194170", - "url": "https://www.northlincs.gov.uk/bins-waste-and-recycling/bin-and-box-collection-dates/", - "wiki_name": "North Lincolnshire Council" - }, - "NorthNorfolkDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "1 Morston Mews", - "postcode": "NR25 6BH", - "url": "https://www.north-norfolk.gov.uk/", - "wiki_name": "North Norfolk District Council", - "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" - }, - "NorthSomersetCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "BS49 5AA", - "uprn": "24051674", - "url": "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule", - "wiki_name": "North Somerset Council" - }, - "NorthTynesideCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "NE26 2TG", - "uprn": "47097627", - "url": "https://my.northtyneside.gov.uk/category/81/bin-collection-dates", - "wiki_name": "North Tyneside Council" - }, - "NorthumberlandCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "22", - "postcode": "NE46 1UQ", - "url": "https://www.northumberland.gov.uk/Waste/Bins/Bin-Calendars.aspx", - "wiki_name": "Northumberland Council" - }, - "PrestonCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "Town Hall", - "postcode": "PR1 2RL", - "url": "https://selfservice.preston.gov.uk/service/Forms/FindMyNearest.aspx?Service=bins", - "wiki_name": "Preston City Council" - }, - "ReigateAndBansteadBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "68134867", - "url": "https://www.reigate-banstead.gov.uk/", - "wiki_name": "Reigate and Banstead Borough Council", - "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" - }, - "RhonddaCynonTaffCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100100778320", - "url": "https://www.rctcbc.gov.uk/EN/Resident/RecyclingandWaste/RecyclingandWasteCollectionDays.aspx", - "wiki_name": "Rhondda Cynon Taff Council", - "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" - }, - "RochdaleCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "OL11 5BE", - "uprn": "23049922", - "url": "https://webforms.rochdale.gov.uk/BinCalendar", - "wiki_name": "Rochdale Council" - }, - "RushcliffeBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "NG13 8TZ", - "uprn": "3040040994", - "url": "https://www.rushcliffe.gov.uk/", - "wiki_name": "Rushcliffe Borough Council" - }, - "RushmoorCouncil": { - "url": "https://www.rushmoor.gov.uk/recycling-rubbish-and-environment/bins-and-recycling/download-or-print-your-bin-collection-calendar/?uprn=XXXXXXXXXX&weeks=16", - "wiki_name": "Rushmoor Council", - "wiki_note": "Replace XXXXXXXXXX in URL with your own UPRN." - }, - "SalfordCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100011416709", - "url": "https://www.salford.gov.uk/bins-and-recycling/bin-collection-days/your-bin-collections", - "wiki_name": "Salford City Council" - }, - "SheffieldCityCouncil": { - "url": "https://wasteservices.sheffield.gov.uk/property/100050931898", - "wiki_command_url_override": "https://wasteservices.sheffield.gov.uk/property/XXXXXXXXXXX", - "wiki_name": "Sheffield City Council", - "wiki_note": "Follow the instructions [here](https://wasteservices.sheffield.gov.uk/) until you get the \"Your bin collection dates and services\" page then copy the URL and replace the URL in the command." - }, - "SomersetCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "TA6 4AA", - "uprn": "10090857775", - "url": "https://www.somerset.gov.uk/", - "wiki_name": "Somerset Council" - }, - "SouthAyrshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "KA19 7BN", - "uprn": "141003134", - "url": "https://www.south-ayrshire.gov.uk/", - "wiki_name": "South Ayrshire Council" - }, - "SouthCambridgeshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "53", - "postcode": "CB23 6GZ", - "url": "https://www.scambs.gov.uk/recycling-and-bins/find-your-household-bin-collection-day/", - "wiki_name": "South Cambridgeshire Council" - }, - "SouthLanarkshireCouncil": { - "url": "https://www.southlanarkshire.gov.uk/directory_record/579973/abbeyhill_crescent_lesmahagow", - "wiki_command_url_override": "https://www.southlanarkshire.gov.uk/directory_record/XXXXX/XXXXX", - "wiki_name": "South Lanarkshire Council", - "wiki_note": "Follow the instructions [here](https://www.southlanarkshire.gov.uk/info/200156/bins_and_recycling/1670/bin_collections_and_calendar) until you get the page that shows the weekly collections for your street then copy the URL and replace the URL in the command." - }, - "SouthNorfolkCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "2630102526", - "url": "https://www.southnorfolkandbroadland.gov.uk/rubbish-recycling/south-norfolk-bin-collection-day-finder", - "wiki_name": "South Norfolk Council" - }, - "SouthOxfordshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "10033002851", - "url": "https://www.southoxon.gov.uk/south-oxfordshire-district-council/recycling-rubbish-and-waste/when-is-your-collection-day/", - "wiki_name": "South Oxfordshire Council" - }, - "SouthTynesideCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "1", - "postcode": "NE33 3JW", - "url": "https://www.southtyneside.gov.uk/article/33352/Bin-collection-dates", - "wiki_name": "South Tyneside Council" - }, - "StHelensBC": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "39081672", - "url": "https://www.sthelens.gov.uk/", - "wiki_name": "St Helens Borough Council" - }, - "StaffordshireMoorlandsDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "ST8 6HN", - "uprn": "100031863037", - "url": "https://www.staffsmoorlands.gov.uk/", - "wiki_name": "Staffordshire Moorlands District Council", - "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" - }, - "StockportBoroughCouncil": { - "url": "https://myaccount.stockport.gov.uk/bin-collections/show/100011434401", - "wiki_command_url_override": "https://myaccount.stockport.gov.uk/bin-collections/show/XXXXXXXX", - "wiki_name": "Stockport Borough Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "StratfordUponAvonCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100070212698", - "url": "https://www.stratford.gov.uk/waste-recycling/when-we-collect.cfm/part/calendar", - "wiki_name": "Stratford Upon Avon Council" - }, - "SwaleBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "ME12 2NQ", - "uprn": "100061081168", - "url": "https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days", - "wiki_name": "Swale Borough Council" - }, - "TamesideMBCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100012835362", - "url": "http://lite.tameside.gov.uk/BinCollections/CollectionService.svc/GetBinCollection", - "wiki_name": "Tameside Metropolitan Borough Council" - }, - "TonbridgeAndMallingBC": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "ME19 4JS", - "uprn": "10002914589", - "url": "https://www.tmbc.gov.uk/", - "wiki_name": "Tonbridge and Malling Borough Council" - }, - "TorbayCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100041053198", - "url": "https://www.torbay.gov.uk/recycling/bin-collections/", - "wiki_name": "Torbay Council" - }, - "TorridgeDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "10091078762", - "url": "https://collections-torridge.azurewebsites.net/WebService2.asmx", - "wiki_name": "Torridge District Council" - }, - "ValeofGlamorganCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "64029020", - "url": "https://www.valeofglamorgan.gov.uk/en/living/Recycling-and-Waste/", - "wiki_name": "Vale of Glamorgan Council" - }, - "WakefieldCityCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "custom_component_show_url_field": true, - "url": "https://www.wakefield.gov.uk/where-i-live/?uprn=63035490&a=115%20Elizabeth%20Drive%20Castleford%20WF10%203RR&usrn=41801243&e=445418&n=426091&p=WF10%203RR", - "wiki_command_url_override": "https://www.wakefield.gov.uk/where-i-live/?uprn=XXXXXXXXXXX&a=XXXXXXXXXXX&usrn=XXXXXXXXXXX&e=XXXXXXXXXXX&n=XXXXXXXXXXX&p=XXXXXXXXXXX", - "wiki_name": "Wakefield City Council", - "wiki_note": "Follow the instructions [here](https://www.wakefield.gov.uk/where-i-live/) until you get the page that includes a \"Bin Collections\" section then copy the URL and replace the URL in the command." - }, - "WarwickDistrictCouncil": { - "url": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/100070263793", - "wiki_command_url_override": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/XXXXXXXX", - "wiki_name": "Warwick District Council", - "wiki_note": "Replace XXXXXXXX with UPRN." - }, - "WaverleyBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "23", - "postcode": "GU9 9QG", - "url": "https://wav-wrp.whitespacews.com/", - "wiki_name": "Waverley Borough Council", - "wiki_note": "Follow the instructions [here](https://wav-wrp.whitespacews.com/#!) until you get the page that shows your next scheduled collections.\nThen take the number from pIndex=NUMBER in the URL and pass it as the -n parameter along with your postcode in -p." - }, - "WealdenDistrictCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "10033413624", - "url": "https://www.wealden.gov.uk/recycling-and-waste/bin-search/", - "wiki_name": "Wealden District Council" - }, - "WelhatCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "AL8 6HQ", - "uprn": "100080982825", - "url": "https://www.welhat.gov.uk/xfp/form/214", - "wiki_name": "Welhat Council" - }, - "WestLothianCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "1 GOSCHEN PLACE", - "postcode": "EH52 5JE", - "url": "https://www.westlothian.gov.uk/", - "wiki_name": "West Lothian Council", - "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" - }, - "WiganBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "WN24UQ", - "uprn": "010093942934", - "url": "https://apps.wigan.gov.uk/MyNeighbourhood/", - "wiki_name": "Wigan Borough Council" - }, - "WiltshireCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "postcode": "SN83TE", - "url": "https://ilambassadorformsprod.azurewebsites.net/wastecollectiondays/index", - "urpn": "100120982570", - "wiki_name": "Wiltshire Council" - }, - "WindsorAndMaidenheadCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "11", - "postcode": "TW19 5NF", - "url": "https://my.rbwm.gov.uk/special/find-your-collection-dates", - "wiki_name": "Windsor and Maidenhead Council" - }, - "WokingBoroughCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "house_number": "2", - "postcode": "GU214JY", - "url": "https://asjwsw-wrpwokingmunicipal-live.whitespacews.com/", - "wiki_name": "Woking Borough Council/Joint Waste Solutions", - "wiki_note": "Works with all collection areas that use Joint Waste Solutions. Just use the correct URL." - }, - "YorkCouncil": { - "SKIP_GET_URL": "SKIP_GET_URL", - "uprn": "100050535540", - "url": "https://waste-api.york.gov.uk/api/Collections/GetBinCollectionDataForUprn/", - "wiki_name": "York Council" - } + "AylesburyValeCouncil": { + "skip_get_url": true, + "uprn": "766252532", + "url": "http://avdcbins.web-labs.co.uk/RefuseApi.asmx", + "wiki_name": "Aylesbury Vale Council (Buckinghamshire)", + "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. Returns all published collections in the past, present, future" + }, + "BCPCouncil": { + "skip_get_url": true, + "uprn": "100040810214", + "url": "https://online.bcpcouncil.gov.uk/bindaylookup/", + "wiki_name": "BCP Council" + }, + "BasingstokeCouncil": { + "skip_get_url": true, + "uprn": "100060220926", + "url": "https://www.basingstoke.gov.uk/bincollection", + "wiki_name": "Basingstoke Council" + }, + "BathAndNorthEastSomersetCouncil": { + "skip_get_url": true, + "uprn": "100120000855", + "url": "https://www.bathnes.gov.uk/webforms/waste/collectionday/", + "wiki_name": "Bath and North East Somerset Council" + }, + "BexleyCouncil": { + "skip_get_url": true, + "uprn": "spamstorage@live.co.uk", + "url": "https://www.bexley.gov.uk/", + "wiki_name": "Bexley Council", + "wiki_note": "In order to use this parser, you will need to sign up to [Bexley's @Home app](https://www.bexley.gov.uk/services/rubbish-and-recycling/bexley-home-recycling-app/about-app) (available for [iOS](https://apps.apple.com/gb/app/home-collection-reminder/id1050703690) and [Android](https://play.google.com/store/apps/details?id=com.contender.athome.android)).\nComplete the setup by entering your email and setting your address with postcode and address line.\nOnce you can see the calendar, you _should_ be good to run the parser.\nJust pass the email you used in quotes in the UPRN parameter.\n" + }, + "BlackburnCouncil": { + "skip_get_url": true, + "uprn": "100010733027", + "url": "https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn=100010733027&month=8&year=2022", + "wiki_command_url_override": "https://www.blackburn.gov.uk", + "wiki_name": "Blackburn Council" + }, + "BoltonCouncil": { + "skip_get_url": true, + "uprn": "100010886949|21 HEATON AVENUE, BOLTON, BL1 5PQ", + "url": "https://www.bolton.gov.uk/next-bin-collection", + "wiki_name": "Bolton Council", + "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN, and the Bolton Council [website](https://www.bolton.gov.uk/next-bin-collection) to find the full address. See [here](https://github.com/robbrad/UKBinCollectionData/issues/272) for more information." + }, + "BristolCityCouncil": { + "skip_get_url": true, + "uprn": "137547", + "url": "https://bristolcouncil.powerappsportals.com/completedynamicformunauth/?servicetypeid=7dce896c-b3ba-ea11-a812-000d3a7f1cdc", + "wiki_name": "Bristol City Council" + }, + "BromleyBoroughCouncil": { + "url": "https://recyclingservices.bromley.gov.uk/waste/6087017", + "wiki_command_url_override": "https://recyclingservices.bromley.gov.uk/waste/XXXXXXX", + "wiki_name": "Bromley Borough Council", + "wiki_note": "Follow the instructions [here](https://recyclingservices.bromley.gov.uk/waste) until the \"Your bin days\" page then copy the URL and replace the URL in the command." + }, + "BroxtoweBoroughCouncil": { + "postcode": "NG16 2LY", + "skip_get_url": true, + "uprn": "100031325997", + "url": "https://www.broxtowe.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "Broxtowe Borough Council" + }, + "BuckinghamshireCouncil": { + "house_number": "HUGHENDEN MANOR, MANOR ROAD, HUGHENDEN VALLEY, HIGH WYCOMBE", + "postcode": "HP14 4LA", + "skip_get_url": true, + "url": "https://chiltern.gov.uk/collection-dates", + "web_driver": "http://selenium:4444", + "wiki_name": "Buckinghamshire Council (Chiltern, South Bucks, Wycombe)", + "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" + }, + "BuryCouncil": { + "house_number": "3", + "postcode": "M26 3XY", + "skip_get_url": true, + "url": "https://www.bury.gov.uk/waste-and-recycling/bin-collection-days-and-alerts", + "wiki_name": "Bury Council", + "wiki_note": "Pass the postcode and house number in their respective arguments, both wrapped in quotes." + }, + "CalderdaleCouncil": { + "postcode": "OL14 7EX", + "skip_get_url": true, + "uprn": "010035034598", + "url": "https://www.calderdale.gov.uk/", + "wiki_name": "Calderdale Council" + }, + "CannockChaseDistrictCouncil": { + "postcode": "WS15 1JA", + "skip_get_url": true, + "uprn": "200003095389", + "url": "https://www.cannockchasedc.gov.uk/", + "wiki_name": "Cannock Chase District Council", + "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" + }, + "CardiffCouncil": { + "skip_get_url": true, + "uprn": "100100112419", + "url": "https://www.cardiff.gov.uk/ENG/resident/Rubbish-and-recycling/When-are-my-bins-collected/Pages/default.aspx", + "wiki_name": "Cardiff Council" + }, + "CastlepointDistrictCouncil": { + "skip_get_url": true, + "uprn": "4525", + "url": "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar", + "wiki_name": "Castlepoint District Council" + }, + "CharnwoodBoroughCouncil": { + "url": "https://my.charnwood.gov.uk/location?put=cbc10070067259&rememberme=0&redirect=%2F", + "wiki_command_url_override": "https://my.charnwood.gov.uk/location?put=cbcXXXXXXXX&rememberme=0&redirect=%2F", + "wiki_name": "Charnwood Borough Council", + "wiki_note": "Replace XXXXXXXX with UPRN keeping \"cbc\" before it." + }, + "ChelmsfordCityCouncil": { + "url": "https://www.chelmsford.gov.uk/myhome/?entry[655e95fb-5f84-4d1b-ab65-878a2f3e3ce4]=14368859", + "wiki_command_url_override": "https://www.chelmsford.gov.uk/myhome/XXXXXX", + "wiki_name": "Chelmsford City Council", + "wiki_note": "Follow the instructions [here](https://www.chelmsford.gov.uk/myhome/) until you get the page listing your \"Address\", \"Ward\" etc then copy the URL and replace the URL in the command." + }, + "CheshireEastCouncil": { + "url": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=100012791226&onelineaddress=3%20COBBLERS%20YARD,%20SK9%207DZ&_=1689413260149", + "wiki_command_url_override": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=XXXXXXXX&onelineaddress=XXXXXXXX&_=1689413260149", + "wiki_name": "Cheshire East Council", + "wiki_note": "Both the UPRN and a one-line address are passed in the URL, which needs to be wrapped in double quotes. The one-line address is made up of the house number, street name and postcode.\nUse the form [here](https://online.cheshireeast.gov.uk/mycollectionday/) to find them, then take the first line and post code and replace all spaces with `%20`." + }, + "ConwyCountyBorough": { + "postcode": "LL30 2DF", + "uprn": "100100429249", + "url": "https://www.conwy.gov.uk/Contensis-Forms/erf/collection-result-soap-xmas.asp?ilangid=1&uprn=100100429249", + "wiki_name": "Conwy County Borough Council", + "wiki_note": "Conwy County Borough Council is a straight up uprn in the url eg &uprn=XXXXXXXXXXXXX ." + }, + "CrawleyBoroughCouncil": { + "house_number": "9701076", + "skip_get_url": true, + "uprn": "100061785321", + "url": "https://my.crawley.gov.uk/", + "wiki_name": "Crawley Borough Council", + "wiki_note": "Crawley needs to be passed both a UPRN and a USRN to work. Find these on [FindMyAddress](https://www.findmyaddress.co.uk/search) or [FindMyStreet](https://www.findmystreet.co.uk/map)." + }, + "CroydonCouncil": { + "house_number": "13", + "postcode": "SE25 5DW", + "skip_get_url": true, + "url": "https://service.croydon.gov.uk/wasteservices/w/webpage/bin-day-enter-address", + "wiki_name": "Croydon Council" + }, + "DerbyshireDalesDistrictCouncil": { + "postcode": "DE4 3AS", + "skip_get_url": true, + "uprn": "10070102161", + "url": "https://www.derbyshiredales.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "Derbyshire Dales District Council" + }, + "DoncasterCouncil": { + "skip_get_url": true, + "uprn": "100050768956", + "url": "https://www.doncaster.gov.uk/Compass/Entity/Launch/D3/", + "wiki_name": "Doncaster Council" + }, + "DurhamCouncil": { + "skip_get_url": true, + "uprn": "200003218818", + "url": "https://www.durham.gov.uk/bincollections?uprn=", + "wiki_name": "Durham Council" + }, + "EastCambridgeshireCouncil": { + "skip_get_url": true, + "uprn": "10002597178", + "url": "https://www.eastcambs.gov.uk/", + "wiki_name": "East Cambridgeshire Council" + }, + "EastDevonDC": { + "url": "https://eastdevon.gov.uk/recycling-and-waste/recycling-and-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=010090909915", + "wiki_command_url_override": "https://eastdevon.gov.uk/recycling-and-waste/recycling-and-waste-information/when-is-my-bin-collected/future-collections-calendar/?UPRN=XXXXXXXX", + "wiki_name": "East Devon District Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "EastLindseyDistrictCouncil": { + "house_number": "Raf Coningsby", + "postcode": "LN4 4SY", + "skip_get_url": true, + "url": "https://www.e-lindsey.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "East Lindsey District Council", + "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" + }, + "EastRidingCouncil": { + "postcode": "HU17 8LG", + "skip_get_url": true, + "url": "https://wasterecyclingapi.eastriding.gov.uk", + "wiki_name": "East Riding Council" + }, + "EastSuffolkCouncil": { + "postcode": "IP11 9FJ", + "skip_get_url": true, + "uprn": "10093544720", + "url": "https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder", + "web_driver": "http://selenium:4444", + "wiki_name": "East Suffolk Council", + "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" + }, + "EastleighBoroughCouncil": { + "skip_get_url": true, + "uprn": "100060303535", + "url": "https://www.eastleigh.gov.uk/waste-bins-and-recycling/collection-dates/your-waste-bin-and-recycling-collections?uprn=", + "wiki_name": "Eastleigh Borough Council" + }, + "ErewashBoroughCouncil": { + "skip_get_url": true, + "uprn": "10003582028", + "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx", + "wiki_name": "Erewash Borough Council" + }, + "FenlandDistrictCouncil": { + "skip_get_url": true, + "uprn": "200002981143", + "url": "https://www.fenland.gov.uk/article/13114/", + "wiki_name": "Fenland District Council" + }, + "GatesheadCouncil": { + "house_number": "Bracken Cottage", + "postcode": "NE16 5LQ", + "skip_get_url": true, + "url": "https://www.gateshead.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "Gateshead Council", + "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" + }, + "GlasgowCityCouncil": { + "url": "https://www.glasgow.gov.uk/forms/refuseandrecyclingcalendar/PrintCalendar.aspx?UPRN=906700034497", + "wiki_command_url_override": "https://www.glasgow.gov.uk/forms/refuseandrecyclingcalendar/PrintCalendar.aspx?UPRN=XXXXXXXX", + "wiki_name": "Glasgow City Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "GuildfordCouncil": { + "skip_get_url": true, + "uprn": "100061372691", + "url": "https://my.guildford.gov.uk/customers/s/sfsites/aura?r=15&other.BinScheduleDisplayCmp.GetBinSchedules=1", + "wiki_name": "Guildford Council", + "wiki_note": "If the bin day is 'today' then the collectionDate will only show today's date if before 7AM, else the date will be in 'previousCollectionDate'. To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." + }, + "HarrogateBoroughCouncil": { + "skip_get_url": true, + "uprn": "100050414307", + "url": "https://secure.harrogate.gov.uk/inmyarea", + "wiki_name": "Harrogate Borough Council", + "wiki_note": "Pass the UPRN which can be found at https://secure.harrogate.gov.uk/inmyarea URL doesn't need to be passed." + }, + "HighPeakCouncil": { + "house_number": "9 Ellison Street, Glossop", + "postcode": "SK13 8BX", + "skip_get_url": true, + "url": "https://www.highpeak.gov.uk/findyourbinday", + "web_driver": "http://selenium:4444", + "wiki_name": "High Peak Council", + "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" + }, + "HuntingdonDistrictCouncil": { + "url": "http://www.huntingdonshire.gov.uk/refuse-calendar/10012048679", + "wiki_command_url_override": "https://www.huntingdonshire.gov.uk/refuse-calendar/XXXXXXXX", + "wiki_name": "Huntingdon District Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "KingstonUponThamesCouncil": { + "url": "https://waste-services.kingston.gov.uk/waste/2701097", + "wiki_command_url_override": "https://waste-services.kingston.gov.uk/waste/XXXXXXX", + "wiki_name": "Kingston Upon Thames Council", + "wiki_note": "Follow the instructions [here](https://waste-services.kingston.gov.uk/waste) until the \"Your bin days\" page then copy the URL and replace the URL in the command." + }, + "LancasterCityCouncil": { + "house_number": "1", + "postcode": "LA1 1RS", + "skip_get_url": true, + "url": "https://lcc-wrp.whitespacews.com", + "wiki_name": "Lancaster City Council" + }, + "LeedsCityCouncil": { + "house_number": "1", + "postcode": "LS6 2SE", + "skip_get_url": true, + "url": "https://www.leeds.gov.uk/residents/bins-and-recycling/check-your-bin-day", + "wiki_name": "Leeds City Council" + }, + "LisburnCastlereaghCityCouncil": { + "house_number": "97", + "postcode": "BT28 1JN", + "skip_get_url": true, + "url": "https://lisburn.isl-fusion.com", + "wiki_name": "Lisburn and Castlereagh City Council" + }, + "LondonBoroughHounslow": { + "skip_get_url": true, + "uprn": "100021577765", + "url": "https://www.hounslow.gov.uk/homepage/86/recycling_and_waste_collection_day_finder", + "wiki_name": "London Borough Hounslow" + }, + "MaldonDistrictCouncil": { + "skip_get_url": true, + "uprn": "100090557253", + "url": "https://maldon.suez.co.uk/maldon/ServiceSummary", + "wiki_name": "Maldon District Council" + }, + "MalvernHillsDC": { + "skip_get_url": true, + "uprn": "100121348457", + "url": "https://swict.malvernhills.gov.uk/mhdcroundlookup/HandleSearchScreen", + "wiki_name": "Malvern Hills District Council" + }, + "ManchesterCityCouncil": { + "skip_get_url": true, + "uprn": "000077048077", + "url": "https://www.manchester.gov.uk/bincollections", + "wiki_name": "Manchester City Council" + }, + "MertonCouncil": { + "url": "https://myneighbourhood.merton.gov.uk/Wasteservices/WasteServices.aspx?ID=28186366", + "wiki_command_url_override": "https://myneighbourhood.merton.gov.uk/Wasteservices/WasteServices.aspx?ID=XXXXXXXX", + "wiki_name": "Merton Council", + "wiki_note": "Follow the instructions [here](https://myneighbourhood.merton.gov.uk/Wasteservices/WasteServicesSearch.aspx) until you get the \"Your recycling and rubbish collection days\" page then copy the URL and replace the URL in the command (the Address parameter is optional)." + }, + "MidSussexDistrictCouncil": { + "house_number": "OAKLANDS", + "postcode": "RH16 1SS", + "skip_get_url": true, + "url": "https://www.midsussex.gov.uk/waste-recycling/bin-collection/", + "wiki_name": "Mid Sussex District Council", + "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" + }, + "MiltonKeynesCityCouncil": { + "skip_get_url": true, + "uprn": "Fullers Slade", + "url": "https://www.milton-keynes.gov.uk/waste-and-recycling/collection-days", + "wiki_name": "Milton Keynes City Council", + "wiki_note": "Pass the name of the estate with the UPRN parameter, wrapped in double quotes" + }, + "NeathPortTalbotCouncil": { + "postcode": "SA13 3BA", + "skip_get_url": true, + "uprn": "10023947752", + "url": "https://www.npt.gov.uk", + "web_driver": "http://selenium:4444", + "wiki_name": "Neath Port Talbot Council" + }, + "NewarkAndSherwoodDC": { + "url": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=200004258529", + "wiki_command_url_override": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX", + "wiki_name": "Newark and Sherwood District Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "NewcastleCityCouncil": { + "url": "https://community.newcastle.gov.uk/my-neighbourhood/ajax/getBinsNew.php?uprn=004510730634", + "wiki_command_url_override": "https://community.newcastle.gov.uk/my-neighbourhood/ajax/getBinsNew.php?uprn=XXXXXXXX", + "wiki_name": "Newcastle City Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "NorthEastLincs": { + "skip_get_url": true, + "uprn": "11062649", + "url": "https://www.nelincs.gov.uk/refuse-collection-schedule/?view=timeline&uprn=11062649", + "wiki_command_url_override": "https://www.nelincs.gov.uk/refuse-collection-schedule/?view=timeline&uprn=XXXXXXXX", + "wiki_name": "North East Lincolnshire Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "NorthKestevenDistrictCouncil": { + "url": "https://www.n-kesteven.org.uk/bins/display?uprn=100030869513", + "wiki_command_url_override": "https://www.n-kesteven.org.uk/bins/display?uprn=XXXXXXXX", + "wiki_name": "North Kesteven District Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "NorthLanarkshireCouncil": { + "url": "https://www.northlanarkshire.gov.uk/bin-collection-dates/000118016164/48402118", + "wiki_command_url_override": "https://www.northlanarkshire.gov.uk/bin-collection-dates/XXXXXXXXXXX/XXXXXXXXXXX", + "wiki_name": "North Lanarkshire Council", + "wiki_note": "Follow the instructions [here](https://www.northlanarkshire.gov.uk/bin-collection-dates) until you get the \"Next collections\" page then copy the URL and replace the URL in the command." + }, + "NorthLincolnshireCouncil": { + "skip_get_url": true, + "uprn": "100050194170", + "url": "https://www.northlincs.gov.uk/bins-waste-and-recycling/bin-and-box-collection-dates/", + "wiki_name": "North Lincolnshire Council" + }, + "NorthNorfolkDistrictCouncil": { + "house_number": "1 Morston Mews", + "postcode": "NR25 6BH", + "skip_get_url": true, + "url": "https://www.north-norfolk.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "North Norfolk District Council", + "wiki_note": "Pass the name of the street with the house number parameter, wrapped in double quotes" + }, + "NorthNorthamptonshireCouncil": { + "skip_get_url": true, + "uprn": "100031021317", + "url": "https://cms.northnorthants.gov.uk/bin-collection-search/calendarevents/100031021318/2023-10-17/2023-10-01", + "wiki_name": "North Northamptonshire Council" + }, + "NorthSomersetCouncil": { + "postcode": "BS49 5AA", + "skip_get_url": true, + "uprn": "24051674", + "url": "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule", + "wiki_name": "North Somerset Council" + }, + "NorthTynesideCouncil": { + "postcode": "NE26 2TG", + "skip_get_url": true, + "uprn": "47097627", + "url": "https://my.northtyneside.gov.uk/category/81/bin-collection-dates", + "wiki_name": "North Tyneside Council" + }, + "NorthumberlandCouncil": { + "house_number": "22", + "postcode": "NE46 1UQ", + "skip_get_url": true, + "url": "https://www.northumberland.gov.uk/Waste/Bins/Bin-Calendars.aspx", + "web_driver": "http://selenium:4444", + "wiki_name": "Northumberland Council" + }, + "PrestonCityCouncil": { + "house_number": "Town Hall", + "postcode": "PR1 2RL", + "skip_get_url": true, + "url": "https://selfservice.preston.gov.uk/service/Forms/FindMyNearest.aspx?Service=bins", + "web_driver": "http://selenium:4444", + "wiki_name": "Preston City Council" + }, + "ReigateAndBansteadBoroughCouncil": { + "skip_get_url": true, + "uprn": "68134867", + "url": "https://www.reigate-banstead.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "Reigate and Banstead Borough Council", + "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" + }, + "RhonddaCynonTaffCouncil": { + "skip_get_url": true, + "uprn": "100100778320", + "url": "https://www.rctcbc.gov.uk/EN/Resident/RecyclingandWaste/RecyclingandWasteCollectionDays.aspx", + "wiki_name": "Rhondda Cynon Taff Council", + "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" + }, + "RochdaleCouncil": { + "postcode": "OL11 5BE", + "skip_get_url": true, + "uprn": "23049922", + "url": "https://webforms.rochdale.gov.uk/BinCalendar", + "wiki_name": "Rochdale Council" + }, + "RushcliffeBoroughCouncil": { + "postcode": "NG13 8TZ", + "skip_get_url": true, + "uprn": "3040040994", + "url": "https://www.rushcliffe.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "Rushcliffe Borough Council" + }, + "RushmoorCouncil": { + "url": "https://www.rushmoor.gov.uk/recycling-rubbish-and-environment/bins-and-recycling/download-or-print-your-bin-collection-calendar/?uprn=XXXXXXXXXX&weeks=16", + "wiki_name": "Rushmoor Council", + "wiki_note": "Replace XXXXXXXXXX in URL with your own UPRN." + }, + "SalfordCityCouncil": { + "skip_get_url": true, + "uprn": "100011416709", + "url": "https://www.salford.gov.uk/bins-and-recycling/bin-collection-days/your-bin-collections", + "wiki_name": "Salford City Council" + }, + "SheffieldCityCouncil": { + "url": "https://wasteservices.sheffield.gov.uk/property/100050931898", + "wiki_command_url_override": "https://wasteservices.sheffield.gov.uk/property/XXXXXXXXXXX", + "wiki_name": "Sheffield City Council", + "wiki_note": "Follow the instructions [here](https://wasteservices.sheffield.gov.uk/) until you get the \"Your bin collection dates and services\" page then copy the URL and replace the URL in the command." + }, + "SomersetCouncil": { + "postcode": "TA6 4AA", + "skip_get_url": true, + "uprn": "10090857775", + "url": "https://www.somerset.gov.uk/", + "wiki_name": "Somerset Council" + }, + "SouthAyrshireCouncil": { + "postcode": "KA19 7BN", + "skip_get_url": true, + "uprn": "141003134", + "url": "https://www.south-ayrshire.gov.uk/", + "wiki_name": "South Ayrshire Council" + }, + "SouthCambridgeshireCouncil": { + "house_number": "53", + "postcode": "CB23 6GZ", + "skip_get_url": true, + "url": "https://www.scambs.gov.uk/recycling-and-bins/find-your-household-bin-collection-day/", + "wiki_name": "South Cambridgeshire Council" + }, + "SouthLanarkshireCouncil": { + "url": "https://www.southlanarkshire.gov.uk/directory_record/579973/abbeyhill_crescent_lesmahagow", + "wiki_command_url_override": "https://www.southlanarkshire.gov.uk/directory_record/XXXXX/XXXXX", + "wiki_name": "South Lanarkshire Council", + "wiki_note": "Follow the instructions [here](https://www.southlanarkshire.gov.uk/info/200156/bins_and_recycling/1670/bin_collections_and_calendar) until you get the page that shows the weekly collections for your street then copy the URL and replace the URL in the command." + }, + "SouthNorfolkCouncil": { + "skip_get_url": true, + "uprn": "2630102526", + "url": "https://www.southnorfolkandbroadland.gov.uk/rubbish-recycling/south-norfolk-bin-collection-day-finder", + "wiki_name": "South Norfolk Council" + }, + "SouthOxfordshireCouncil": { + "skip_get_url": true, + "uprn": "10033002851", + "url": "https://www.southoxon.gov.uk/south-oxfordshire-district-council/recycling-rubbish-and-waste/when-is-your-collection-day/", + "wiki_name": "South Oxfordshire Council" + }, + "SouthTynesideCouncil": { + "house_number": "1", + "postcode": "NE33 3JW", + "skip_get_url": true, + "url": "https://www.southtyneside.gov.uk/article/33352/Bin-collection-dates", + "wiki_name": "South Tyneside Council" + }, + "StHelensBC": { + "skip_get_url": true, + "uprn": "39081672", + "url": "https://www.sthelens.gov.uk/", + "wiki_name": "St Helens Borough Council" + }, + "StaffordshireMoorlandsDistrictCouncil": { + "postcode": "ST8 6HN", + "skip_get_url": true, + "uprn": "100031863037", + "url": "https://www.staffsmoorlands.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "Staffordshire Moorlands District Council", + "wiki_note": "To get the UPRN, you can use [FindMyAddress](https://www.findmyaddress.co.uk/search)" + }, + "StockportBoroughCouncil": { + "url": "https://myaccount.stockport.gov.uk/bin-collections/show/100011434401", + "wiki_command_url_override": "https://myaccount.stockport.gov.uk/bin-collections/show/XXXXXXXX", + "wiki_name": "Stockport Borough Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "StratfordUponAvonCouncil": { + "skip_get_url": true, + "uprn": "100070212698", + "url": "https://www.stratford.gov.uk/waste-recycling/when-we-collect.cfm/part/calendar", + "wiki_name": "Stratford Upon Avon Council" + }, + "SwaleBoroughCouncil": { + "postcode": "ME12 2NQ", + "skip_get_url": true, + "uprn": "100061081168", + "url": "https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days", + "wiki_name": "Swale Borough Council" + }, + "TamesideMBCouncil": { + "skip_get_url": true, + "uprn": "100012835362", + "url": "http://lite.tameside.gov.uk/BinCollections/CollectionService.svc/GetBinCollection", + "wiki_name": "Tameside Metropolitan Borough Council" + }, + "TonbridgeAndMallingBC": { + "postcode": "ME19 4JS", + "skip_get_url": true, + "uprn": "10002914589", + "url": "https://www.tmbc.gov.uk/", + "wiki_name": "Tonbridge and Malling Borough Council" + }, + "TorbayCouncil": { + "skip_get_url": true, + "uprn": "100041053198", + "url": "https://www.torbay.gov.uk/recycling/bin-collections/", + "wiki_name": "Torbay Council" + }, + "TorridgeDistrictCouncil": { + "skip_get_url": true, + "uprn": "10091078762", + "url": "https://collections-torridge.azurewebsites.net/WebService2.asmx", + "wiki_name": "Torridge District Council" + }, + "ValeofGlamorganCouncil": { + "skip_get_url": true, + "uprn": "64029020", + "url": "https://www.valeofglamorgan.gov.uk/en/living/Recycling-and-Waste/", + "wiki_name": "Vale of Glamorgan Council" + }, + "WakefieldCityCouncil": { + "custom_component_show_url_field": true, + "skip_get_url": true, + "url": "https://www.wakefield.gov.uk/where-i-live/?uprn=63035490&a=115%20Elizabeth%20Drive%20Castleford%20WF10%203RR&usrn=41801243&e=445418&n=426091&p=WF10%203RR", + "web_driver": "http://selenium:4444", + "wiki_command_url_override": "https://www.wakefield.gov.uk/where-i-live/?uprn=XXXXXXXXXXX&a=XXXXXXXXXXX&usrn=XXXXXXXXXXX&e=XXXXXXXXXXX&n=XXXXXXXXXXX&p=XXXXXXXXXXX", + "wiki_name": "Wakefield City Council", + "wiki_note": "Follow the instructions [here](https://www.wakefield.gov.uk/where-i-live/) until you get the page that includes a \"Bin Collections\" section then copy the URL and replace the URL in the command." + }, + "WarwickDistrictCouncil": { + "url": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/100070263793", + "wiki_command_url_override": "https://estates7.warwickdc.gov.uk/PropertyPortal/Property/Recycling/XXXXXXXX", + "wiki_name": "Warwick District Council", + "wiki_note": "Replace XXXXXXXX with UPRN." + }, + "WaverleyBoroughCouncil": { + "house_number": "23", + "postcode": "GU9 9QG", + "skip_get_url": true, + "url": "https://wav-wrp.whitespacews.com/", + "wiki_name": "Waverley Borough Council", + "wiki_note": "Follow the instructions [here](https://wav-wrp.whitespacews.com/#!) until you get the page that shows your next scheduled collections.\nThen take the number from pIndex=NUMBER in the URL and pass it as the -n parameter along with your postcode in -p." + }, + "WealdenDistrictCouncil": { + "skip_get_url": true, + "uprn": "10033413624", + "url": "https://www.wealden.gov.uk/recycling-and-waste/bin-search/", + "wiki_name": "Wealden District Council" + }, + "WelhatCouncil": { + "postcode": "AL8 6HQ", + "skip_get_url": true, + "uprn": "100080982825", + "url": "https://www.welhat.gov.uk/xfp/form/214", + "wiki_name": "Welhat Council" + }, + "WestLothianCouncil": { + "house_number": "1 GOSCHEN PLACE", + "postcode": "EH52 5JE", + "skip_get_url": true, + "url": "https://www.westlothian.gov.uk/", + "web_driver": "http://selenium:4444", + "wiki_name": "West Lothian Council", + "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" + }, + "WiganBoroughCouncil": { + "postcode": "WN24UQ", + "skip_get_url": true, + "uprn": "010093942934", + "url": "https://apps.wigan.gov.uk/MyNeighbourhood/", + "wiki_name": "Wigan Borough Council" + }, + "WiltshireCouncil": { + "postcode": "SN83TE", + "skip_get_url": true, + "url": "https://ilambassadorformsprod.azurewebsites.net/wastecollectiondays/index", + "urpn": "100120982570", + "wiki_name": "Wiltshire Council" + }, + "WindsorAndMaidenheadCouncil": { + "house_number": "11", + "postcode": "TW19 5NF", + "skip_get_url": true, + "url": "https://my.rbwm.gov.uk/special/find-your-collection-dates", + "wiki_name": "Windsor and Maidenhead Council" + }, + "WokingBoroughCouncil": { + "house_number": "2", + "postcode": "GU214JY", + "skip_get_url": true, + "url": "https://asjwsw-wrpwokingmunicipal-live.whitespacews.com/", + "wiki_name": "Woking Borough Council/Joint Waste Solutions", + "wiki_note": "Works with all collection areas that use Joint Waste Solutions. Just use the correct URL." + }, + "YorkCouncil": { + "skip_get_url": true, + "uprn": "100050535540", + "url": "https://waste-api.york.gov.uk/api/Collections/GetBinCollectionDataForUprn/", + "wiki_name": "York Council" + } } \ No newline at end of file diff --git a/uk_bin_collection/tests/step_defs/test_validate_council.py b/uk_bin_collection/tests/step_defs/test_validate_council.py index 3fc8289a6d..2d26fe6efc 100644 --- a/uk_bin_collection/tests/step_defs/test_validate_council.py +++ b/uk_bin_collection/tests/step_defs/test_validate_council.py @@ -50,7 +50,11 @@ def scrape_step(context, council): if "usrn" in context.metadata: usrn = context.metadata["usrn"] args.append(f"-us={usrn}") - if "SKIP_GET_URL" in context.metadata: + # TODO we should somehow run this test with and without this argument passed + if "web_driver" in context.metadata: + web_driver = context.metadata["web_driver"] + args.append(f"-w={web_driver}") + if "skip_get_url" in context.metadata: args.append(f"-s") try: diff --git a/uk_bin_collection/tests/test_common_functions.py b/uk_bin_collection/tests/test_common_functions.py index b73bae4209..d95dba6274 100644 --- a/uk_bin_collection/tests/test_common_functions.py +++ b/uk_bin_collection/tests/test_common_functions.py @@ -161,35 +161,38 @@ def test_get_next_occurrence_from_day_month_true(): result = get_next_occurrence_from_day_month(datetime(2023,9,1)) assert result == pd.Timestamp('2024-09-01 00:00:00') -def test_write_output_json(): +def test_update_input_json(): council = "test_council" - content = '{"example": "data"}' - write_output_json(council, content) + url = "TEST_URL" + postcode="TEST_POSTCODE" + uprn="TEST_UPRN" + web_driver="TEST_WEBDRIVER" + skip_get_url = True + update_input_json(council, url, postcode=postcode, uprn=uprn, web_driver=web_driver, skip_get_url=skip_get_url) cwd = os.getcwd() - outputs_path = os.path.join(cwd, "uk_bin_collection", "tests", "outputs", council + ".json") - result1 = os.path.exists(outputs_path) - - with open(outputs_path, "r") as f: - read_content = f.read() - - if os.path.exists(outputs_path): - os.remove(outputs_path) - + input_file_path = os.path.join(cwd, "uk_bin_collection", "tests", "input.json") + result1 = os.path.exists(input_file_path) + with open(input_file_path, 'r') as f: + data = json.load(f) assert result1 == True - assert read_content == content + assert data[council] == {"postcode": postcode, "skip_get_url": skip_get_url, "uprn": uprn, "url": url, "web_driver": web_driver, "wiki_name": council} -def test_write_output_json_fail(capsys, monkeypatch): +def test_update_input_json_fail(capsys, monkeypatch): def mock_os_path_exists(path): return False # Simulate the path not existing monkeypatch.setattr(os.path, 'exists', mock_os_path_exists) council = "test_council" - content = '{"example": "data"}' - write_output_json(council, content) + url = "TEST_URL" + postcode="TEST_POSTCODE" + uprn="TEST_UPRN" + web_driver="TEST_WEBDRIVER" + skip_get_url = True + update_input_json(council, url, postcode=postcode, uprn=uprn, web_driver=web_driver, skip_get_url=skip_get_url) captured = capsys.readouterr() - assert "Exception encountered: Unable to save Output JSON file for the council." in captured.out + assert "Exception encountered: Unable to update input.json file for the council." in captured.out assert "Please check you're running developer mode" in captured.out def test_create_webdriver(): diff --git a/uk_bin_collection/uk_bin_collection/collect_data.py b/uk_bin_collection/uk_bin_collection/collect_data.py index 4494c39d2c..7952741a45 100644 --- a/uk_bin_collection/uk_bin_collection/collect_data.py +++ b/uk_bin_collection/uk_bin_collection/collect_data.py @@ -35,6 +35,12 @@ def __init__(self): required=False, ) self.parser.add_argument("-u", "--uprn", type=str, help="UPRN to parse", required=False) + self.parser.add_argument( + "-w", + "--web_driver", + help="URL for remote Selenium web driver - should be wrapped in double quotes", + required=False, + ) self.parser.add_argument( "-d", "--dev_mode", @@ -66,6 +72,7 @@ def run(self): paon = self.parsed_args.number uprn = self.parsed_args.uprn skip_get_url = self.parsed_args.skip_get_url + web_driver = self.parsed_args.web_driver dev_mode = self.parsed_args.dev_mode return self.client_code( @@ -75,6 +82,7 @@ def run(self): paon=paon, uprn=uprn, skip_get_url=skip_get_url, + web_driver=web_driver, dev_mode=dev_mode, council_module_str=council_module_str, ) diff --git a/uk_bin_collection/uk_bin_collection/common.py b/uk_bin_collection/uk_bin_collection/common.py index 7143d0446c..56d5231d1d 100644 --- a/uk_bin_collection/uk_bin_collection/common.py +++ b/uk_bin_collection/uk_bin_collection/common.py @@ -200,16 +200,43 @@ def remove_alpha_characters(input_string: str) -> str: return "".join(c for c in input_string if c.isdigit() or c == " ") -def write_output_json(council: str, content: str): +def update_input_json(council: str, url: str, **kwargs): + """ + Create/update council's entry in the input.json + :param council: Council + :param kwargs: Run arguments + """ + postcode = kwargs.get("postcode", None) + paon = kwargs.get("paon", None) + uprn = kwargs.get("uprn", None) + usrn = kwargs.get("usrn", None) + web_driver = kwargs.get("web_driver", None) + skip_get_url = kwargs.get("skip_get_url", None) cwd = os.getcwd() - outputs_path = os.path.join(cwd, "..", "tests", "outputs") - if not os.path.exists(outputs_path) or not os.path.isdir(outputs_path): - outputs_path = os.path.join(cwd, "uk_bin_collection", "tests", "outputs") - if os.path.exists(outputs_path) and os.path.isdir(outputs_path): - with open(os.path.join(outputs_path, council + ".json"), "w") as f: - f.write(content) + input_file_path = os.path.join(cwd, "uk_bin_collection", "tests", "input.json") + if os.path.exists(input_file_path): + with open(input_file_path, 'r') as f: + data = json.load(f) + if council not in data: + data[council] = {"wiki_name": council} + if "url" != "": + data[council]["url"] = url + if postcode is not None: + data[council]["postcode"] = postcode + if paon is not None: + data[council]["paon"] = paon + if uprn is not None: + data[council]["uprn"] = uprn + if usrn is not None: + data[council]["usrn"] = usrn + if web_driver is not None: + data[council]["web_driver"] = web_driver + if skip_get_url is not None: + data[council]["skip_get_url"] = skip_get_url + with open(input_file_path, 'w') as f: + f.write(json.dumps(data, sort_keys=True, indent=4)) else: - print("Exception encountered: Unable to save Output JSON file for the council.") + print("Exception encountered: Unable to update input.json file for the council.") print( "Please check you're running developer mode from either the UKBinCollectionData " "or uk_bin_collection/uk_bin_collection/ directories." @@ -221,7 +248,7 @@ def validate_dates(bin_dates: dict) -> dict: # If a date is in December and the next is in January, increase the year -def create_webdriver() -> webdriver.Chrome: +def create_webdriver(web_driver) -> webdriver.Chrome: """ Create and return a headless Selenium webdriver :rtype: webdriver.Chrome @@ -233,5 +260,8 @@ def create_webdriver() -> webdriver.Chrome: options.add_argument("--disable-gpu") options.add_argument("--disable-dev-shm-usage") options.add_experimental_option("excludeSwitches", ["enable-logging"]) - # Return a Selenium webdriver + # Return a remote Selenium webdriver + if web_driver is not None: + return webdriver.Remote(command_executor=web_driver, options=options) + # Return a local Selenium webdriver return webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options) diff --git a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py index 1614a100b0..c3861c4b80 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py index aa034adec4..4b8dae8caf 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py @@ -37,9 +37,10 @@ def parse_data(self, page: str, **kwargs) -> dict: # Assign user info user_postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") + web_driver = kwargs.get("web_driver") # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # Enter postcode in text box and wait diff --git a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py index 4fe73a18e6..c48ed87b0b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py index 24e0cae5f7..4b6a345760 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get("https://www.e-lindsey.gov.uk/article/6714/Your-Waste-Collection-Days") # Wait for the postcode field to appear then populate it diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index cd7f61f1f2..d1ce7a4501 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -19,11 +19,12 @@ class CouncilClass(AbstractGetBinDataClass): def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get("https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder") # Wait for iframe to load and switch to it diff --git a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py index 471274120c..042292f6e1 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get("https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker") # Wait for the postcode field to appear then populate it diff --git a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py index 9ff3c9907a..9fcc184813 100644 --- a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py @@ -59,9 +59,10 @@ def parse_data(self, page: str, **kwargs) -> dict: # Assign user info user_postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") + web_driver = kwargs.get("web_driver") # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # Hide Cookies diff --git a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py index 394cd69441..1588e856c1 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py @@ -22,11 +22,12 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get("https://www.npt.gov.uk/2195") # Accept cookies banner diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py index a6dc199140..f2fecc9689 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py index 69e36407f1..6f4f4a62e4 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py @@ -32,11 +32,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) time.sleep(1) diff --git a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py index d1aae0f698..c0f2621c12 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py @@ -27,11 +27,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py index 0aa3f70b02..6f03b862e2 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py @@ -18,12 +18,13 @@ class CouncilClass(AbstractGetBinDataClass): def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) # Pad UPRN with 0's at the start for any that aren't 12 chars user_uprn = user_uprn.zfill(12) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get( f"https://my.reigate-banstead.gov.uk/en/service/Bins_and_recycling___collections_calendar?uprn={user_uprn}") diff --git a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py index 3144d13918..316cf773dd 100644 --- a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py index 190bbc5988..a25f6b7ede 100644 --- a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py @@ -21,11 +21,12 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get("https://www.staffsmoorlands.gov.uk/findyourbinday") # Close cookies banner diff --git a/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py index a020799137..bf53660cf3 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py @@ -15,7 +15,7 @@ class CouncilClass(AbstractGetBinDataClass): def parse_data(self, page: str, **kwargs) -> dict: # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(kwargs.get("web_driver")) driver.get(kwargs.get("url")) # Make a BS4 object diff --git a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py index 3d0eaf6822..da50b87f91 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") + web_driver = kwargs.get("web_driver") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver() + driver = create_webdriver(web_driver) driver.get("https://www.westlothian.gov.uk/article/31528/Bin-Collection-Calendar-Dates") # Close feedback banner diff --git a/uk_bin_collection/uk_bin_collection/get_bin_data.py b/uk_bin_collection/uk_bin_collection/get_bin_data.py index 2ea59c9f37..604a880369 100644 --- a/uk_bin_collection/uk_bin_collection/get_bin_data.py +++ b/uk_bin_collection/uk_bin_collection/get_bin_data.py @@ -10,7 +10,7 @@ import requests -from uk_bin_collection.uk_bin_collection.common import write_output_json +from uk_bin_collection.uk_bin_collection.common import update_input_json LOGGING_CONFIG = dict( version=1, @@ -55,6 +55,7 @@ def template_method(self, address_url: str, **kwargs) -> None: # pragma: no cov this_paon = kwargs.get("paon", None) this_uprn = kwargs.get("uprn", None) this_usrn = kwargs.get("usrn", None) + this_web_driver = kwargs.get("web_driver", None) skip_get_url = kwargs.get("skip_get_url", None) dev_mode = kwargs.get("dev_mode", False) council_module_str = kwargs.get("council_module_str", None) @@ -63,18 +64,18 @@ def template_method(self, address_url: str, **kwargs) -> None: # pragma: no cov ): # we will not use the generic way to get data - needs a get data in the council class itself page = self.get_data(address_url) bin_data_dict = self.parse_data( - page, postcode=this_postcode, paon=this_paon, uprn=this_uprn, usrn=this_usrn, url=this_url + page, postcode=this_postcode, paon=this_paon, uprn=this_uprn, usrn=this_usrn, web_driver=this_web_driver, url=this_url ) json_output = self.output_json(bin_data_dict) else: bin_data_dict = self.parse_data( - "", postcode=this_postcode, paon=this_paon, uprn=this_uprn, usrn=this_usrn, url=this_url + "", postcode=this_postcode, paon=this_paon, uprn=this_uprn, usrn=this_usrn, web_driver=this_web_driver, url=this_url ) json_output = self.output_json(bin_data_dict) - # if dev mode create/update council's output JSON if bin_data_dict is not empty - if dev_mode is not None and dev_mode is True and bin_data_dict["bins"]: - write_output_json(council_module_str, json_output) + # if dev mode create/update council's entry in the input.json + if dev_mode is not None and dev_mode is True: + update_input_json(council_module_str, this_url, postcode=this_postcode, paon=this_paon, uprn=this_uprn, usrn=this_usrn, web_driver=this_web_driver, skip_get_url=skip_get_url) return json_output diff --git a/wiki/generate_wiki.py b/wiki/generate_wiki.py index b65018ea89..235697a842 100644 --- a/wiki/generate_wiki.py +++ b/wiki/generate_wiki.py @@ -46,6 +46,9 @@ def main(): if "usrn" in council_details: command += " -usrn XXXXXXXX" additional_parameters += "- `-us` - USRN\n" + if "web_driver" in council_details: + command += " -w http://HOST:PORT/" + additional_parameters += "- `-w` - remote Selenium web driver URL (required for Home Assistant)\n" # add to entries entries += "\n---\n\n" From 2a1d1c8ff6161c4e3f4003290fa6fe623e482824 Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Thu, 2 Nov 2023 21:41:23 +0000 Subject: [PATCH 2/5] feat: Add remote Selenium web driver support --- .../uk_bin_collection/councils/BroxtoweBoroughCouncil.py | 3 +++ .../uk_bin_collection/councils/BuckinghamshireCouncil.py | 3 +++ .../councils/DerbyshireDalesDistrictCouncil.py | 3 +++ .../uk_bin_collection/councils/EastLindseyDistrictCouncil.py | 3 +++ .../uk_bin_collection/councils/EastSuffolkCouncil.py | 3 +++ .../uk_bin_collection/councils/GatesheadCouncil.py | 3 +++ .../uk_bin_collection/councils/HighPeakCouncil.py | 3 +++ .../uk_bin_collection/councils/NeathPortTalbotCouncil.py | 3 +++ .../uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py | 3 +++ .../uk_bin_collection/councils/NorthumberlandCouncil.py | 3 +++ .../uk_bin_collection/councils/PrestonCityCouncil.py | 3 +++ .../councils/ReigateAndBansteadBoroughCouncil.py | 3 +++ .../uk_bin_collection/councils/RushcliffeBoroughCouncil.py | 3 +++ .../councils/StaffordshireMoorlandsDistrictCouncil.py | 3 +++ .../uk_bin_collection/councils/WakefieldCityCouncil.py | 3 +++ .../uk_bin_collection/councils/WestLothianCouncil.py | 3 +++ 16 files changed, 48 insertions(+) diff --git a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py index c3861c4b80..748afe9728 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py @@ -61,6 +61,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + bins_div = soup.find("div", id="ctl00_ContentPlaceHolder1_FF5686FormGroup") if bins_div: bins_table = bins_div.find("table") diff --git a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py index 4b8dae8caf..8928480b05 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py @@ -78,6 +78,9 @@ def parse_data(self, page: str, **kwargs) -> dict: df = pd.read_html(table, header=[1]) df = df[0] + # Quit Selenium webdriver to release session + driver.quit() + # Parse data into dict data = self.get_data(df) diff --git a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py index c48ed87b0b..883daebb90 100644 --- a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py @@ -61,6 +61,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + bin_rows = soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation") \ .find("div", {"class": "row"}).find_all("div", {"class": "row"}) if bin_rows: diff --git a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py index 4b6a345760..d646d32ddd 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py @@ -60,6 +60,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + # Get collections for collection in soup.find_all("div", {"class": "waste-result"}): ptags = collection.find_all("p") diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index d1ce7a4501..e149bcc4fb 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -67,6 +67,9 @@ def parse_data(self, page: str, **kwargs) -> dict: ) ) + # Quit Selenium webdriver to release session + driver.quit() + # Make a BS4 object soup = BeautifulSoup(data_table.get_attribute("innerHTML"), features="html.parser") diff --git a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py index 042292f6e1..0c875de2cb 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py @@ -53,6 +53,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + # Get collections table table = soup.find("table", {"class": "bincollections__table"}) diff --git a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py index 9fcc184813..94cf20c188 100644 --- a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py @@ -113,6 +113,9 @@ def parse_data(self, page: str, **kwargs) -> dict: By.ID, "FINDBINDAYSHIGHPEAK_CALENDAR_MAINCALENDAR" ).get_attribute("outerHTML") + # Quit Selenium webdriver to release session + driver.quit() + # Parse data into dict data = self.get_data(table) diff --git a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py index 1588e856c1..50cb795b4f 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py @@ -77,6 +77,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + # Get the property details property_details = soup.find( "div", diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py index f2fecc9689..f516720f6a 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py @@ -62,6 +62,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + bins_text = soup.find("div", id="Search_result_details_cps_hd") if bins_text: diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py index 6f4f4a62e4..b547a26bc4 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py @@ -64,6 +64,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + # Work out which bins can be collected for this address. Glass bins are only on some houses due to pilot programme. bins_collected = list( map( diff --git a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py index c0f2621c12..f045c1bf5b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py @@ -67,6 +67,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + topLevelSpan = soup.find( "span", id="lblCollectionDates" diff --git a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py index 6f03b862e2..87c3f66178 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py @@ -39,6 +39,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") soup.prettify() + # Quit Selenium webdriver to release session + driver.quit() + data = {"bins": []} section = soup.find("span", {"data-name": "html2"}) dates = section.find_all("div") diff --git a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py index 316cf773dd..8b1c1e8903 100644 --- a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py @@ -61,6 +61,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + bins_text = soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation") if bins_text: diff --git a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py index a25f6b7ede..0cdc176c79 100644 --- a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py @@ -68,6 +68,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + # Get months for month_wrapper in soup.find_all("div", {"class": "bin-collection__month"}): if month_wrapper: diff --git a/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py index bf53660cf3..eaa1d551b8 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WakefieldCityCouncil.py @@ -22,6 +22,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") soup.prettify() + # Quit Selenium webdriver to release session + driver.quit() + data = {"bins": []} sections = soup.find_all("div", {"class": "wil_c-content-section_heading"}) for s in sections: diff --git a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py index da50b87f91..e49b0ca66d 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py @@ -60,6 +60,9 @@ def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(driver.page_source, features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + # Get collections for collection in soup.find_all("div", {"class": "bin-collect"}): dict_data = { From 4178354251e7e412e771be2568648e11c452e981 Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Thu, 2 Nov 2023 21:46:15 +0000 Subject: [PATCH 3/5] feat: Add remote Selenium web driver support --- uk_bin_collection/tests/test_common_functions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/uk_bin_collection/tests/test_common_functions.py b/uk_bin_collection/tests/test_common_functions.py index d95dba6274..fc2b991624 100644 --- a/uk_bin_collection/tests/test_common_functions.py +++ b/uk_bin_collection/tests/test_common_functions.py @@ -195,6 +195,10 @@ def mock_os_path_exists(path): assert "Exception encountered: Unable to update input.json file for the council." in captured.out assert "Please check you're running developer mode" in captured.out -def test_create_webdriver(): - result = create_webdriver() +def test_create_webdriver_local(): + result = create_webdriver(None) + assert result.name == 'chrome' + +def test_create_webdriver_remote(): + result = create_webdriver("http://selenium:4444") assert result.name == 'chrome' From 11bf496fb2f49a24736557f38b281d9113292503 Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Fri, 3 Nov 2023 08:17:39 +0000 Subject: [PATCH 4/5] feat: Add remote Selenium web driver support --- .github/workflows/behave.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml index 953161c8ea..e073737f1b 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave.yml @@ -17,11 +17,6 @@ on: schedule: - cron: '0 0 * * *' -services: - selenium: - image: selenium/standalone-chrome:latest - options: --shm-size=2gb - jobs: build: runs-on: ubuntu-latest @@ -29,6 +24,10 @@ jobs: matrix: python-version: ['3.10', 3.11] poetry-version: [1.5.1] + services: + selenium: + image: selenium/standalone-chrome:latest + options: --shm-size=2gb steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 From e9ea8d168c9809b761ef7cd73839acadcba9c02a Mon Sep 17 00:00:00 2001 From: Oliver Cullimore Date: Fri, 3 Nov 2023 08:25:13 +0000 Subject: [PATCH 5/5] feat: Add remote Selenium web driver support --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9fc8da28f..f1740f8ed7 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Most scripts make use of [Beautiful Soup 4](https://pypi.org/project/beautifulso 1. Click `+ Add Integration` and search for and select `UK Bin Collection Data` then click `Download`. 1. Restart your Home Assistant. 1. In the Home Assistant UI go to `Settings` > `Devices & Services` click `+ Add Integration` and search for `UK Bin Collection Data`. +1. If your see a "URL of the remote Selenium web driver to use" field when setting up your council, you'll need to provide the URL to a web driver you've set up seperately such as [standalone-chrome](https://hub.docker.com/r/selenium/standalone-chrome). ### Install manually