Merge pull request #397 from OliverCullimore/fix-ha-custom-component

pyproject.toml

@@ -55,6 +55,7 @@ requests = "*"
 selenium = "*"
 lxml = "*"
 urllib3 = "*"
+webdriver-manager = "^4.0.1"
 
 [tool.commitizen]
 major_version_zero = true

uk_bin_collection/uk_bin_collection/common.py

@@ -1,13 +1,15 @@
 import calendar
+import holidays
 import json
 import os
+import pandas as pd
 import re
+import requests
 from datetime import datetime
 from enum import Enum
-
-import holidays
-import pandas as pd
-import requests
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service as ChromeService
+from webdriver_manager.chrome import ChromeDriverManager
 
 date_format = "%d/%m/%Y"
 days_of_week = {
@@ -217,3 +219,19 @@ def write_output_json(council: str, content: str):
 def validate_dates(bin_dates: dict) -> dict:
     raise NotImplementedError()
     # If a date is in December and the next is in January, increase the year
+
+
+def create_webdriver() -> webdriver.Chrome:
+    """
+    Create and return a headless Selenium webdriver
+    :rtype: webdriver.Chrome
+    """
+    # Set up Selenium to run 'headless'
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-gpu")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_experimental_option("excludeSwitches", ["enable-logging"])
+    # Return a Selenium webdriver
+    return webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options)

uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py

@@ -1,9 +1,9 @@
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -27,16 +27,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_uprn(user_uprn)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         # Populate postcode field

uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py

@@ -1,11 +1,9 @@
-import time
-
 import pandas as pd
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
+import time
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support.ui import Select
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -40,16 +38,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         user_postcode = kwargs.get("postcode")
         user_paon = kwargs.get("paon")
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         # Enter postcode in text box and wait

uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py

@@ -1,9 +1,9 @@
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -27,16 +27,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_uprn(user_uprn)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         # Populate postcode field

uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py

@@ -1,5 +1,4 @@
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
@@ -24,16 +23,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_paon(user_paon)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get("https://www.e-lindsey.gov.uk/article/6714/Your-Waste-Collection-Days")
 
         # Wait for the postcode field to appear then populate it

uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py

@@ -1,8 +1,8 @@
 from bs4 import BeautifulSoup
-from selenium import webdriver
-from selenium.webdriver.support.ui import WebDriverWait, Select
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait, Select
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -17,21 +17,13 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         user_uprn = kwargs.get("uprn")
         user_postcode = kwargs.get("postcode")
         check_uprn(user_uprn)
         check_postcode(user_postcode)
 
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get("https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder")
 
         # Wait for iframe to load and switch to it
@@ -42,7 +34,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
             EC.presence_of_element_located((By.ID, "alt_postcode_search"))
         )
         # Enter postcode
-        postcode.send_keys(user_postcode)
+        postcode.send_keys(user_postcode.replace(" ", ""))
 
         # Wait for address selection dropdown to appear
         address = Select(
@@ -74,8 +66,6 @@ def parse_data(self, page: str, **kwargs) -> dict:
             )
         )
 
-
-
         # Make a BS4 object
         soup = BeautifulSoup(data_table.get_attribute("innerHTML"), features="html.parser")
 

uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py

@@ -1,5 +1,4 @@
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
@@ -24,16 +23,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_paon(user_paon)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get("https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker")
 
         # Wait for the postcode field to appear then populate it

uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py

@@ -1,12 +1,10 @@
-import time
-
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.support.wait import WebDriverWait
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -62,16 +60,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         user_postcode = kwargs.get("postcode")
         user_paon = kwargs.get("paon")
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         # Hide Cookies

uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py

@@ -1,6 +1,5 @@
 import time
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
@@ -26,16 +25,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_uprn(user_uprn)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get("https://www.npt.gov.uk/2195")
 
         # Accept cookies banner

uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py

@@ -1,9 +1,9 @@
 from bs4 import BeautifulSoup
-from selenium import webdriver
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -27,16 +27,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_paon(user_paon)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         # Populate postcode field

uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py

@@ -1,16 +1,12 @@
 import time
-
-import requests
 from bs4 import BeautifulSoup
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.support.ui import Select
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
 
+
 # import the wonderful Beautiful Soup and the URL grabber
 
 
@@ -39,16 +35,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_paon(user_paon)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         time.sleep(1)

uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py

@@ -1,13 +1,10 @@
-from datetime import datetime
-
 from bs4 import BeautifulSoup
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
+from datetime import datetime
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import \
     AbstractGetBinDataClass
@@ -33,16 +30,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_paon(user_paon)
         check_postcode(user_postcode)
 
-        # Set up Selenium to run 'headless'
-        options = webdriver.ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("excludeSwitches", ["enable-logging"])
-
         # Create Selenium webdriver
-        driver = webdriver.Chrome(options=options)
+        driver = create_webdriver()
         driver.get(page)
 
         # If you bang in the house number (or property name) and postcode in the box it should find your property