From df94e7729503e3146918abb2c177ffe5f704d276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20=C3=98rb=C3=A6k=20Chemnitz?= Date: Tue, 21 Nov 2023 09:14:53 +0100 Subject: [PATCH 1/4] Handle unknown authorization error with status 200 --- twscrape/queue_client.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index e40356e..50415bd 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -38,11 +38,14 @@ class RateLimitError(Exception): class BannedError(Exception): pass - class DependencyError(Exception): pass +class UnknownAuthorizationError(Exception): + pass + + def req_id(rep: httpx.Response): lr = str(rep.headers.get("x-rate-limit-remaining", -1)) ll = str(rep.headers.get("x-rate-limit-limit", -1)) @@ -170,6 +173,9 @@ async def _check_rep(self, rep: httpx.Response): if rep.status_code == 200 and "_Missing: No status found with that ID." in msg: return # ignore this error + if rep.status_code == 200 and "Authorization: Denied by access control: unspecified reason" in msg: + raise UnknownAuthorizationError(msg) + # todo: (32) Could not authenticate you if msg != "OK": @@ -197,7 +203,10 @@ async def req(self, method: str, url: str, params: ReqParams = None): # already handled continue except DependencyError: - logger.error(f"Dependency error, returnning: {url}") + logger.error(f"Dependency error, returning: {url}") + return + except UnknownAuthorizationError: + logger.error(f"Unknown authorization error, returning: {url}") return except (httpx.ReadTimeout, httpx.ProxyError): # http transport failed, just retry From 24bd19cb018a573097bfafe7518580c4f0f497ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20=C3=98rb=C3=A6k=20Chemnitz?= Date: Thu, 21 Dec 2023 23:05:51 +0100 Subject: [PATCH 2/4] banned on unknown auth error --- twscrape/queue_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index 50415bd..ca1a9f1 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -173,7 +173,8 @@ async def _check_rep(self, rep: httpx.Response): if rep.status_code == 200 and "_Missing: No status found with that ID." in msg: return # ignore this error - if rep.status_code == 200 and "Authorization: Denied by access control: unspecified reason" in msg: + if rep.status_code == 200 and "Authorization" in msg: + await self._close_ctx(-1, banned=True, msg=msg) raise UnknownAuthorizationError(msg) # todo: (32) Could not authenticate you From b514e181a1da42c4efb0501a90a2a36ff797b574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20=C3=98rb=C3=A6k=20Chemnitz?= Date: Thu, 21 Dec 2023 23:08:30 +0100 Subject: [PATCH 3/4] not banned, but broader unknown auth error --- twscrape/queue_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index ca1a9f1..a032a84 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -174,7 +174,6 @@ async def _check_rep(self, rep: httpx.Response): return # ignore this error if rep.status_code == 200 and "Authorization" in msg: - await self._close_ctx(-1, banned=True, msg=msg) raise UnknownAuthorizationError(msg) # todo: (32) Could not authenticate you From 87acabb76bedf570a5167b3c03436a45761910bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20=C3=98rb=C3=A6k=20Chemnitz?= Date: Fri, 22 Dec 2023 08:03:22 +0100 Subject: [PATCH 4/4] not banned, but broader unknown auth error --- twscrape/queue_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index a032a84..535fd3a 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -174,12 +174,15 @@ async def _check_rep(self, rep: httpx.Response): return # ignore this error if rep.status_code == 200 and "Authorization" in msg: + return # ignore this error raise UnknownAuthorizationError(msg) # todo: (32) Could not authenticate you if msg != "OK": - raise ApiError(rep, res) + logger.error(f"Unknown error: {msg}") + return # ignore this error + #raise ApiError(rep, res) rep.raise_for_status()