Merge pull request #470 from robbrad/176_North_East_Derbyshire_Distri…

…ct_Council

uk_bin_collection/tests/features/validate_council_outputs.feature

@@ -66,6 +66,7 @@ Feature: Test each council output matches expected results
             | NeathPortTalbotCouncil | http://selenium:4444 | local |
             | NewarkAndSherwoodDC | None | None |
             | NewcastleCityCouncil | None | None |
+            | NorthEastDerbyshireDistrictCouncil | http://selenium:4444 | local |
             | NorthEastLincs | None | None |
             | NorthKestevenDistrictCouncil | None | None |
             | NorthLanarkshireCouncil | None | None |

uk_bin_collection/tests/input.json

@@ -393,6 +393,14 @@
         "wiki_command_url_override": "https://community.newcastle.gov.uk/my-neighbourhood/ajax/getBinsNew.php?uprn=XXXXXXXX",
         "wiki_name": "Newcastle City Council",
         "wiki_note": "Replace XXXXXXXX with UPRN."
+    },    
+    "NorthEastDerbyshireDistrictCouncil": {
+        "skip_get_url": true,
+        "uprn": "010034492221",
+        "postcode": "S42 5RB",
+        "url": "https://myselfservice.ne-derbyshire.gov.uk/service/Check_your_Bin_Day",
+        "wiki_name": "North East Derbyshire District Council",
+        "web_driver": "http://selenium:4444"
     },
     "NorthEastLincs": {
         "skip_get_url": true,
@@ -425,7 +433,7 @@
         "uprn": "100030572613",
         "postcode": "DE74 2FZ",
         "url": "https://nwleics-self.achieveservice.com/en/AchieveForms/?form_uri=sandbox-publish://AF-Process-bde93819-fa47-4bba-b094-bef375dbef0c/AF-Stage-b4ac5d55-7fb7-4c40-809f-4d1856399bed/definition.json&redirectlink=/en&cancelRedirectLink=/en&noLoginPrompt=1",
-        "wiki_name": "NorthWest Leicestershire Council",
+        "wiki_name": "North West Leicestershire Council",
         "web_driver": "http://selenium:4444"
     },
     "NorthNorfolkDistrictCouncil": {

uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py

@@ -0,0 +1,96 @@
+from bs4 import BeautifulSoup
+from datetime import datetime
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        page = "https://myselfservice.ne-derbyshire.gov.uk/service/Check_your_Bin_Day"
+
+        data = {"bins": []}
+
+        user_uprn = kwargs.get("uprn")
+        user_postcode = kwargs.get("postcode")
+        web_driver = kwargs.get("web_driver")
+        check_uprn(user_uprn)
+        check_postcode(user_postcode)
+        # Create Selenium webdriver
+        driver = create_webdriver(web_driver)
+        driver.get(page)
+
+        # If you bang in the house number (or property name) and postcode in the box it should find your property
+
+        iframe_presense = WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located((By.ID, "fillform-frame-1"))
+        )
+
+        driver.switch_to.frame(iframe_presense)
+        wait = WebDriverWait(driver, 60)
+        inputElement_postcodesearch = wait.until(
+            EC.element_to_be_clickable((By.NAME, "postcode_search"))
+        )
+
+        inputElement_postcodesearch.send_keys(str(user_postcode))
+
+        # Wait for the 'Select your property' dropdown to appear and select the first result
+        dropdown = wait.until(EC.element_to_be_clickable((By.NAME, "selAddress")))
+
+        dropdown_options = wait.until(
+            EC.presence_of_element_located((By.CLASS_NAME, "lookup-option"))
+        )
+
+        # Create a 'Select' for it, then select the first address in the list
+        # (Index 0 is "Make a selection from the list")
+        drop_down_values = Select(dropdown)
+        option_element = wait.until(
+            EC.presence_of_element_located((By.CSS_SELECTOR, f'option.lookup-option[value="{str(user_uprn)}"]'))
+        )
+
+        drop_down_values.select_by_value(str(user_uprn))
+
+        # Wait for the 'View more' link to appear, then click it to get the full set of dates
+        h3_element = wait.until(
+            EC.presence_of_element_located((By.XPATH, "//th[contains(text(), 'Waste Collection')]"))
+        )
+
+        soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+        target_h3 = soup.find('h3', string='Collection Details')
+        tables_after_h3 = target_h3.parent.parent.find_next('table')
+
+        table_rows = tables_after_h3.find_all("tr")
+        for row in table_rows:
+            rowdata = row.find_all("td")
+            if len(rowdata) == 3:
+                labels = rowdata[0].find_all('label')
+                # Strip the day (i.e., Monday) out of the collection date string for parsing
+                if len(labels) >= 2:
+                    date_label = labels[1]
+                    datestring = date_label.text.strip()
+
+                # Add the bin type and collection date to the 'data' dictionary
+                data["bins"].append(
+                    {
+                        "type": rowdata[2].text.strip(),
+                        "collectionDate": datetime.strptime(
+                            datestring, "%d/%m/%Y"
+                        ).strftime(date_format),  # Format the date as needed
+                    }
+                )
+
+        return data