From 02cebedff42c568ce43dd69bafebe0948432e473 Mon Sep 17 00:00:00 2001 From: Ben Fayers Date: Fri, 3 Nov 2023 20:30:49 +0000 Subject: [PATCH 1/3] feat: Driver quit needs to be after last use of driver --- .../uk_bin_collection/councils/EastSuffolkCouncil.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index e149bcc4fb..e85ee96c81 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -67,12 +67,13 @@ def parse_data(self, page: str, **kwargs) -> dict: ) ) - # Quit Selenium webdriver to release session - driver.quit() - + # Make a BS4 object soup = BeautifulSoup(data_table.get_attribute("innerHTML"), features="html.parser") + # Quit Selenium webdriver to release session + driver.quit() + data = {"bins": []} rows = soup.find("table").find("tbody").find_all("tr") From 22a269b4cb6825e2bba955ef9ffc864942e4520a Mon Sep 17 00:00:00 2001 From: Ben Fayers Date: Fri, 3 Nov 2023 20:31:40 +0000 Subject: [PATCH 2/3] feat: Change bin_type's to be title() so it reads better --- .../uk_bin_collection/councils/EastSuffolkCouncil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index e85ee96c81..ab08f68ab3 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -79,7 +79,7 @@ def parse_data(self, page: str, **kwargs) -> dict: rows = soup.find("table").find("tbody").find_all("tr") for row in rows: cols = row.find_all("td") - bin_type = cols[2].find_all("span")[1].text + bin_type = cols[2].find_all("span")[1].text.title() collection_date = cols[3].find_all("span")[1].text collection_date = datetime.strptime(collection_date, "%d/%m/%Y").strftime( date_format From eda70935d84d9383b9083fd3b69c445e58abbeac Mon Sep 17 00:00:00 2001 From: Ben Fayers Date: Fri, 3 Nov 2023 20:33:53 +0000 Subject: [PATCH 3/3] feat: Update EastSuffolkCouncil.py --- .../uk_bin_collection/councils/EastSuffolkCouncil.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index ab08f68ab3..975130d32a 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -1,3 +1,4 @@ +from time import sleep from bs4 import BeautifulSoup from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -49,6 +50,9 @@ def parse_data(self, page: str, **kwargs) -> dict: EC.invisibility_of_element_located((By.CLASS_NAME, "spinner-outer")) ) + # Sometimes the options aren't fully there despite the spinner being gone, wait another 2 seconds. + sleep(2) + # Select address by UPRN address.select_by_value(user_uprn) @@ -57,6 +61,8 @@ def parse_data(self, page: str, **kwargs) -> dict: EC.invisibility_of_element_located((By.CLASS_NAME, "spinner-outer")) ) + sleep(2) + # Find data table data_table = WebDriverWait(driver, 10).until( EC.presence_of_element_located(