feat: Council Pack 9

uk_bin_collection/tests/input.json

@@ -484,6 +484,20 @@
         "url": "https://www.fenland.gov.uk/article/13114/",
         "wiki_name": "Fenland District Council"
     },
+    "FifeCouncil": {
+        "url": "https://www.fife.gov.uk",
+        "wiki_command_url_override": "https://www.fife.gov.uk",
+        "uprn": "320203521",
+        "wiki_name": "Fife Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
+    "FlintshireCountyCouncil": {
+        "url": "https://digital.flintshire.gov.uk",
+        "wiki_command_url_override": "https://digital.flintshire.gov.uk",
+        "uprn": "100100213710",
+        "wiki_name": "Flintshire County Council",
+        "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
+    },
     "ForestOfDeanDistrictCouncil": {
         "house_number": "ELMOGAL, PARKEND ROAD, BREAM, LYDNEY",
         "postcode": "GL15 6JT",
@@ -1143,8 +1157,8 @@
         "wiki_name": "South Tyneside Council"
     },
     "SouthwarkCouncil": {
-        "url": "https://www.southwark.gov.uk/bins/lookup/",
-        "wiki_command_url_override": "https://www.southwark.gov.uk/bins/lookup/XXXXXXXX",
+        "url": "https://services.southwark.gov.uk/bins/lookup/",
+        "wiki_command_url_override": "https://services.southwark.gov.uk/bins/lookup/XXXXXXXX",
         "uprn": "200003469271",
         "wiki_name": "Southwark Council",
         "wiki_note": "Replace XXXXXXXX with UPRN. You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
@@ -1182,6 +1196,14 @@
         "wiki_name": "Stockport Borough Council",
         "wiki_note": "Replace XXXXXXXX with UPRN."
     },
+    "StocktonOnTeesCouncil": {
+        "house_number": "24",
+        "postcode": "TS20 2RD",
+        "skip_get_url": true,
+        "url": "https://www.stockton.gov.uk",
+        "web_driver": "http://selenium:4444",
+        "wiki_name": "Stockton On Tees Council"
+    },
     "StokeOnTrentCityCouncil": {
         "url": "https://www.stoke.gov.uk/jadu/custom/webserviceLookUps/BarTecWebServices_missed_bin_calendar.php?UPRN=3455121482",
         "wiki_command_url_override": "https://www.stoke.gov.uk/jadu/custom/webserviceLookUps/BarTecWebServices_missed_bin_calendar.php?UPRN=XXXXXXXXXX",

uk_bin_collection/uk_bin_collection/councils/FifeCouncil.py

@@ -0,0 +1,68 @@
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        # Get and check UPRN
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        API_URL = "https://www.fife.gov.uk/api/custom?action=powersuite_bin_calendar_collections&actionedby=bin_calendar&loadform=true&access=citizen&locale=en"
+        AUTH_URL = "https://www.fife.gov.uk/api/citizen?preview=false&locale=en"
+        AUTH_KEY = "Authorization"
+
+        r = requests.get(AUTH_URL)
+        r.raise_for_status()
+        auth_token = r.headers[AUTH_KEY]
+
+        post_data = {
+            "name": "bin_calendar",
+            "data": {
+                "uprn": user_uprn,
+            },
+            "email": "",
+            "caseid": "",
+            "xref": "",
+            "xref1": "",
+            "xref2": "",
+        }
+
+        headers = {
+            "referer": "https://www.fife.gov.uk/services/forms/bin-calendar",
+            "accept": "application/json",
+            "content-type": "application/json",
+            AUTH_KEY: auth_token,
+        }
+
+        r = requests.post(API_URL, data=json.dumps(post_data), headers=headers)
+        r.raise_for_status()
+
+        result = r.json()
+
+        for collection in result["data"]["tab_collections"]:
+            dict_data = {
+                "type": collection["colour"],
+                "collectionDate": datetime.strptime(
+                    collection["date"],
+                    "%A, %B %d, %Y",
+                ).strftime("%d/%m/%Y"),
+            }
+            bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata

uk_bin_collection/uk_bin_collection/councils/FlintshireCountyCouncil.py

@@ -0,0 +1,60 @@
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+        bindata = {"bins": []}
+
+        URI = f"https://digital.flintshire.gov.uk/FCC_BinDay/Home/Details2/{user_uprn}"
+
+        # Make the GET request
+        response = requests.get(URI)
+
+        # Parse the HTML content
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        # Adjust these tags and classes based on actual structure
+        # Example for finding collection dates and types
+        bin_collections = soup.find_all(
+            "div", class_="col-md-12 col-lg-12 col-sm-12 col-xs-12"
+        )  # Replace with actual class name
+
+        # Extracting and printing the schedule data
+        schedule = []
+        for collection in bin_collections:
+            dates = collection.find_all("div", class_="col-lg-2 col-md-2 col-sm-2")
+            bin_type = collection.find("div", class_="col-lg-3 col-md-3 col-sm-3")
+
+            if dates[0].text.strip() == "Date of Collection":
+                continue
+
+            bin_types = bin_type.text.strip().split(" / ")
+            date = dates[0].text.strip()
+
+            # Loop through the dates for each collection type
+            for bin_type in bin_types:
+
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": date,
+                }
+                bindata["bins"].append(dict_data)
+
+        bindata["bins"].sort(
+            key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")
+        )
+        return bindata

uk_bin_collection/uk_bin_collection/councils/SouthwarkCouncil.py

@@ -19,7 +19,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_uprn(user_uprn)
         data = {"bins": []}
 
-        baseurl = "https://www.southwark.gov.uk/bins/lookup/"
+        baseurl = "https://services.southwark.gov.uk/bins/lookup/"
         url = baseurl + user_uprn
 
         headers = {
@@ -74,9 +74,13 @@ def parse_data(self, page: str, **kwargs) -> dict:
             data["bins"].append(dict_data)
 
         # Extract food waste collection information
-        food_section = soup.find("div", {"aria-labelledby": "organicsCollectionTitle"})
+        food_section = soup.find(
+            "div", {"aria-labelledby": "domesticFoodCollectionTitle"}
+        )
         if food_section:
-            food_title = food_section.find("p", {"id": "organicsCollectionTitle"}).text
+            food_title = food_section.find(
+                "p", {"id": "domesticFoodCollectionTitle"}
+            ).text
             food_next_collection = (
                 food_section.find(text=lambda text: "Next collection" in text)
                 .strip()

uk_bin_collection/uk_bin_collection/councils/StocktonOnTeesCouncil.py

@@ -0,0 +1,159 @@
+import time
+
+from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            data = {"bins": []}
+            collections = []
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_paon(user_paon)
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get("https://www.stockton.gov.uk/bin-collection-days")
+
+            # Wait for the postcode field to appear then populate it
+            inputElement_postcode = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_ADDRESSLOOKUPPOSTCODE",
+                    )
+                )
+            )
+            inputElement_postcode.send_keys(user_postcode)
+
+            # Click search button
+            findAddress = WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located(
+                    (
+                        By.ID,
+                        "LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_ADDRESSLOOKUPSEARCH",
+                    )
+                )
+            )
+            findAddress.click()
+
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(
+                    (
+                        By.XPATH,
+                        ""
+                        "//*[@id='LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_ADDRESSLOOKUPADDRESS']//option[contains(., '"
+                        + user_paon
+                        + "')]",
+                    )
+                )
+            ).click()
+
+            # Wait for the submit button to appear, then click it to get the collection dates
+            WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        '//*[@id="LOOKUPBINDATESBYADDRESSSKIPOUTOFREGION_COLLECTIONDETAILS2"]/div',
+                    )
+                )
+            )
+            time.sleep(2)
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+            soup.prettify()
+
+            rubbish_div = soup.find(
+                "p",
+                {
+                    "class": "myaccount-block__date myaccount-block__date--bin myaccount-block__date--waste"
+                },
+            )
+            rubbish_date = rubbish_div.text
+            if rubbish_date == "Today":
+                rubbish_date = datetime.now()
+            else:
+                rubbish_date = datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(rubbish_date).strip(),
+                    "%a %d %B %Y",
+                ).replace(year=datetime.now().year)
+
+            recycling_div = soup.find(
+                "p",
+                {
+                    "class": "myaccount-block__date myaccount-block__date--bin myaccount-block__date--recycling"
+                },
+            )
+            recycling_date = recycling_div.text
+            if recycling_date == "Today":
+                recycling_date = datetime.now()
+            else:
+                recycling_date = datetime.strptime(
+                    remove_ordinal_indicator_from_date_string(recycling_date).strip(),
+                    "%a %d %B %Y",
+                )
+
+            garden_div = soup.find(
+                "div",
+                {
+                    "class": "myaccount-block__item myaccount-block__item--bin myaccount-block__item--garden"
+                },
+            )
+            garden_date = garden_div.find("strong")
+            if garden_date.text.strip() == "Date not available":
+                print("Garden waste unavailable")
+            else:
+                if garden_date.text == "Today":
+                    garden_date = datetime.now()
+                    collections.append(("Garden waste bin", garden_date))
+                else:
+                    garden_date = datetime.strptime(
+                        remove_ordinal_indicator_from_date_string(
+                            garden_date.text
+                        ).strip(),
+                        "%a %d %B %Y",
+                    )
+                    collections.append(("Garden waste bin", garden_date))
+
+            collections.append(("Rubbish bin", rubbish_date))
+            collections.append(("Recycling bin", recycling_date))
+
+            ordered_data = sorted(collections, key=lambda x: x[1])
+            for item in ordered_data:
+                dict_data = {
+                    "type": item[0].capitalize(),
+                    "collectionDate": item[1].strftime(date_format),
+                }
+                data["bins"].append(dict_data)
+
+            print()
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return data