-
Notifications
You must be signed in to change notification settings - Fork 100
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #867 from m26dvd/master
feat: Council Pack 2
- Loading branch information
Showing
5 changed files
with
403 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
uk_bin_collection/uk_bin_collection/councils/ElmbridgeBoroughCouncil.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| import time | ||
|
|
||
| import requests | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| # import the wonderful Beautiful Soup and the URL grabber | ||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Concrete classes have to implement all abstract operations of the | ||
| base class. They can also override some operations with a default | ||
| implementation. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
|
|
||
| user_uprn = kwargs.get("uprn") | ||
| check_uprn(user_uprn) | ||
| bindata = {"bins": []} | ||
|
|
||
| BASE_URL = "https://elmbridge-self.achieveservice.com" | ||
| INTIAL_URL = f"{BASE_URL}/service/Your_bin_collection_days" | ||
| AUTH_URL = f"{BASE_URL}/authapi/isauthenticated" | ||
| AUTH_TEST = f"{BASE_URL}/apibroker/domain/elmbridge-self.achieveservice.com" | ||
| API_URL = f"{BASE_URL}/apibroker/runLookup" | ||
|
|
||
| self._session = requests.Session() | ||
| r = self._session.get(INTIAL_URL) | ||
| r.raise_for_status() | ||
| params: dict[str, str | int] = { | ||
| "uri": r.url, | ||
| "hostname": "elmbridge-self.achieveservice.com", | ||
| "withCredentials": "true", | ||
| } | ||
| r = self._session.get(AUTH_URL, params=params) | ||
| r.raise_for_status() | ||
| data = r.json() | ||
| session_key = data["auth-session"] | ||
|
|
||
| params = { | ||
| "sid": session_key, | ||
| "_": int(datetime.now().timestamp() * 1000), | ||
| } | ||
| r = self._session.get(AUTH_TEST, params=params) | ||
| r.raise_for_status() | ||
|
|
||
| params: dict[str, int | str] = { | ||
| "id": "663b557cdaece", | ||
| "repeat_against": "", | ||
| "noRetry": "false", | ||
| "getOnlyTokens": "undefined", | ||
| "log_id": "", | ||
| "app_name": "AF-Renderer::Self", | ||
| "_": int(datetime.now().timestamp() * 1000), | ||
| "sid": session_key, | ||
| } | ||
| payload = { | ||
| "formValues": { | ||
| "Section 1": { | ||
| "UPRN": {"value": user_uprn}, | ||
| } | ||
| } | ||
| } | ||
| r = self._session.post(API_URL, params=params, json=payload) | ||
| r.raise_for_status() | ||
|
|
||
| data = r.json() | ||
| collections = list(r.json()["integration"]["transformed"]["rows_data"].values()) | ||
|
|
||
| for collection in collections: | ||
| date = collection["Date"] | ||
| for service in [ | ||
| collection["Service1"], | ||
| collection["Service2"], | ||
| collection["Service3"], | ||
| ]: | ||
| if not service: | ||
| continue | ||
| service = service.removesuffix(" Collection Service") | ||
|
|
||
| dict_data = { | ||
| "type": service, | ||
| "collectionDate": datetime.strptime( | ||
| date, "%d/%m/%Y %H:%M:%S" | ||
| ).strftime("%d/%m/%Y"), | ||
| } | ||
| bindata["bins"].append(dict_data) | ||
|
|
||
| return bindata |
88 changes: 88 additions & 0 deletions
88
uk_bin_collection/uk_bin_collection/councils/HighlandCouncil.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| import time | ||
|
|
||
| import requests | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| # import the wonderful Beautiful Soup and the URL grabber | ||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Concrete classes have to implement all abstract operations of the | ||
| base class. They can also override some operations with a default | ||
| implementation. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
|
|
||
| user_uprn = kwargs.get("uprn") | ||
| check_uprn(user_uprn) | ||
| bindata = {"bins": []} | ||
|
|
||
| SESSION_URL = "https://highland-self.achieveservice.com/authapi/isauthenticated?uri=https%3A%2F%2Fhighland-self.achieveservice.com%2Fen%2Fservice%2FCheck_your_household_bin_collection_days&hostname=highland-self.achieveservice.com&withCredentials=true" | ||
|
|
||
| API_URL = "https://highland-self.achieveservice.com/apibroker/runLookup" | ||
|
|
||
| data = { | ||
| "formValues": {"Your address": {"propertyuprn": {"value": user_uprn}}}, | ||
| } | ||
| headers = { | ||
| "Content-Type": "application/json", | ||
| "Accept": "application/json", | ||
| "User-Agent": "Mozilla/5.0", | ||
| "X-Requested-With": "XMLHttpRequest", | ||
| "Referer": "https://highland-self.achieveservice.com/fillform/?iframe_id=fillform-frame-1&db_id=", | ||
| } | ||
| s = requests.session() | ||
| r = s.get(SESSION_URL) | ||
| r.raise_for_status() | ||
| session_data = r.json() | ||
| sid = session_data["auth-session"] | ||
| params = { | ||
| "id": "660d44a698632", | ||
| "repeat_against": "", | ||
| "noRetry": "false", | ||
| "getOnlyTokens": "undefined", | ||
| "log_id": "", | ||
| "app_name": "AF-Renderer::Self", | ||
| # unix_timestamp | ||
| "_": str(int(time.time() * 1000)), | ||
| "sid": sid, | ||
| } | ||
|
|
||
| r = s.post(API_URL, json=data, headers=headers, params=params) | ||
| r.raise_for_status() | ||
|
|
||
| data = r.json() | ||
| rows_data = data["integration"]["transformed"]["rows_data"]["0"] | ||
| if not isinstance(rows_data, dict): | ||
| raise ValueError("Invalid data returned from API") | ||
|
|
||
| use_new = any(k.endswith("New") and v for k, v in rows_data.items()) | ||
| next_date_key = "NextDateNew" if use_new else "NextDateOld" | ||
|
|
||
| for key, value in rows_data.items(): | ||
| if not (key.endswith("NextDate") or key.endswith(next_date_key)): | ||
| continue | ||
|
|
||
| bin_type = key.split("NextDate")[0] | ||
| if bin_type == "refuse": | ||
| bin_type = "Non-recyclable waste" | ||
| if bin_type == "fibres": | ||
| bin_type = "Paper, card and cardboard recycling" | ||
| if bin_type == "containers": | ||
| bin_type = "Plastics, metals and cartons recycling" | ||
| if bin_type == "garden": | ||
| bin_type = "Garden waste" | ||
| if bin_type == "food": | ||
| bin_type = "Food waste" | ||
|
|
||
| try: | ||
| date = datetime.strptime(value, "%Y-%m-%d").strftime("%d/%m/%Y") | ||
| except ValueError: | ||
| continue | ||
| dict_data = {"type": bin_type, "collectionDate": date} | ||
| bindata["bins"].append(dict_data) | ||
|
|
||
| return bindata |
60 changes: 60 additions & 0 deletions
60
uk_bin_collection/uk_bin_collection/councils/SouthDerbyshireDistrictCouncil.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| import requests | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| # import the wonderful Beautiful Soup and the URL grabber | ||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Concrete classes have to implement all abstract operations of the | ||
| base class. They can also override some operations with a default | ||
| implementation. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
|
|
||
| user_uprn = kwargs.get("uprn") | ||
| check_uprn(user_uprn) | ||
| data = {"bins": []} | ||
|
|
||
| baseurl = "https://maps.southderbyshire.gov.uk/iShareLIVE.web//getdata.aspx?RequestType=LocalInfo&ms=mapsources/MyHouse&format=JSONP&group=Recycling%20Bins%20and%20Waste|Next%20Bin%20Collections&uid=" | ||
| url = baseurl + user_uprn | ||
|
|
||
| # Make the web request | ||
| response = requests.get(url).text | ||
|
|
||
| # Remove the JSONP wrapper using a regular expression | ||
| jsonp_pattern = r"\{.*\}" | ||
| json_match = re.search(jsonp_pattern, response) | ||
|
|
||
| if json_match: | ||
| # Extract the JSON part | ||
| json_data = json_match.group(0) | ||
|
|
||
| # Parse the JSON | ||
| parsed_data = json.loads(json_data) | ||
|
|
||
| # Extract the embedded HTML string | ||
| html_content = parsed_data["Results"]["Next_Bin_Collections"]["_"] | ||
|
|
||
| # Parse the HTML to extract dates and bin types using regex | ||
| matches = re.findall( | ||
| r"<span.*?>(\d{2} \w+ \d{4})</span>.*?<span.*?>(.*?)</span>", | ||
| html_content, | ||
| re.S, | ||
| ) | ||
|
|
||
| # Output the parsed bin collection details | ||
| for match in matches: | ||
| dict_data = { | ||
| "type": match[1], | ||
| "collectionDate": datetime.strptime(match[0], "%d %B %Y").strftime( | ||
| "%d/%m/%Y" | ||
| ), | ||
| } | ||
| data["bins"].append(dict_data) | ||
| else: | ||
| print("No valid JSON found in the response.") | ||
|
|
||
| return data |
Oops, something went wrong.